Merge branch 'master' of https://github.com/GeminiDRSoftware/DRAGONS

.travis.yml

@@ -0,0 +1,35 @@
+os: linux
+dist: xenial
+language: python
+python:
+  # We don't actually use the Travis Python, but this keeps it organized.
+  - "3.7"
+env:
+  global:
+    - DRAGONS_TEST="$HOME/dragons_tests/"
+install:
+  # We do this conditionally because it saves us some downloading if the
+  # version is the same.
+  - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
+  - bash miniconda.sh -b -p $HOME/miniconda
+  - source "$HOME/miniconda/etc/profile.d/conda.sh"
+  - hash -r
+  - conda config --set always_yes yes --set changeps1 no
+  - conda update --quiet conda
+  - conda config --add channels http://ssb.stsci.edu/astroconda
+  - conda config --add channels http://astroconda.gemini.edu/public/noarch
+  - conda config --set channel_priority false
+  - conda config --set restore_free_channel true
+  - conda env list
+  - conda install --yes pip wheel
+  - pip install "tox<=3.17.1" tox-conda
+
+  # Useful for debugging any issues with conda
+  - conda info -a
+
+script:
+  - tox -e py37-unit -v
+
+after_success:
+  - pip install codecov
+  - codecov

astrodata/testing.py

@@ -223,9 +223,23 @@ def change_working_dir(path_to_outputs):
     os.makedirs(path, exist_ok=True)
 
     @contextmanager
-    def _change_working_dir():
+    def _change_working_dir(sub_path=""):
+        """
+        Changed the current working directory temporarily easily using the 
+        `with` statement.
+         
+        Parameters
+        ----------
+        sub_path : str 
+            Sub-path inside the directory where we are working.
+        """
         oldpwd = os.getcwd()
         os.chdir(path)
+
+        if sub_path:
+            os.makedirs(sub_path, exist_ok=True)
+            os.chdir(sub_path)
+
         try:
             yield
         finally:

astrodata/tests/test_testing.py

@@ -12,6 +12,28 @@
 from astrodata.testing import assert_same_class, download_from_archive
 
 
+def test_change_working_dir(change_working_dir):
+    """
+    Test the change_working_dir fixture.
+
+    Parameters
+    ----------
+    change_working_dir : fixture
+        Custom DRAGONS fixture.
+    """
+    assert "astrodata/test_testing/outputs" not in os.getcwd()
+
+    with change_working_dir():
+        assert "astrodata/test_testing/outputs" in os.getcwd()
+
+    assert "astrodata/test_testing/outputs" not in os.getcwd()
+
+    with change_working_dir("my_sub_dir"):
+        assert "astrodata/test_testing/outputs/my_sub_dir" in os.getcwd()
+
+    assert "astrodata/test_testing/outputs" not in os.getcwd()
+
+
 def test_download_from_archive_raises_ValueError_if_envvar_does_not_exists():
     with pytest.raises(ValueError):
         download_from_archive('N20180304S0126.fits', env_var='')

geminidr/gmos/recipes/qa/tests/test_flat_image.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python
+"""
+Tests the GMOS Image Bias reduction for QA mode.
+"""
+import os
+import pytest
+
+import astrodata
+import gemini_instruments
+import numpy as np
+
+from astrodata.testing import download_from_archive
+from gempy.utils import logutils
+from recipe_system.reduction.coreReduce import Reduce
+from recipe_system.utils.reduce_utils import normalize_ucals
+
+test_data = [
+    # GMOS-N HeII
+    ("N20190101S0494_bias.fits",
+     ["N20181201S{:04d}.fits".format(n) for n in range(283, 286)]),
+]
+
+
+@pytest.mark.integtest
+@pytest.mark.parametrize("master_bias, flat_fnames", test_data)
+def test_make_processed_flat(
+        change_working_dir, flat_fnames, master_bias, path_to_inputs):
+    """
+    Regression test for GMOS
+
+    Parameters
+    ----------
+    change_working_dir : fixture
+        Custom fixture defined astrodata.testing that changes temporarily the
+        working dir.
+    flat_fnames : list
+        Contains the flat names that will be reduced.
+    master_bias : str
+        Contains the name of the master flat.
+    path_to_inputs : fixture
+        Custom fixture that defines where the input data is stored.
+    """
+    master_bias = os.path.join(path_to_inputs, master_bias)
+    calibration_files = ['processed_bias:{}'.format(master_bias)]
+
+    with change_working_dir():
+        logutils.config(file_name=f"log_flat_{flat_fnames[0].split('.')[0]}.txt")
+        r = Reduce()
+        r.files = [download_from_archive(f) for f in flat_fnames]
+        r.mode = 'qa'
+        r.ucals = normalize_ucals(r.files, calibration_files)
+        r.runr()
+
+        ad = astrodata.open(r.output_filenames[0])
+
+    for ext in ad:
+        data = np.ma.masked_array(ext.data, mask=ext.mask)
+
+        if not data.mask.all():
+            np.testing.assert_allclose(np.ma.median(data.ravel()), 1, atol=0.071)
+            np.testing.assert_allclose(data[~data.mask], 1, atol=0.45)
+
+    # plot(ad)
+
+
+# -- Helper functions ----------------------------------------------------------
+def plot(ad):
+    """
+    Displays the tiled arrays with the DQ mask for analysing the data.
+
+    Parameters
+    ----------
+    ad : multi-extension data
+    """
+    import numpy as np
+    import matplotlib.pyplot as plt
+
+    from astropy.visualization import ImageNormalize, ZScaleInterval
+    from copy import deepcopy
+    from geminidr.gmos.primitives_gmos_image import GMOSImage
+
+    p = GMOSImage([deepcopy(ad)])
+    _ad = p.tileArrays().pop()
+
+    fig, axs = plt.subplots(num=ad.filename, ncols=len(_ad), sharey=True)
+
+    norm = ImageNormalize(
+        np.concatenate([ext.data.ravel()[ext.mask.ravel() == 0] for ext in _ad]),
+        interval=ZScaleInterval())
+
+    vmin = norm.vmin
+    vmax = norm.vmax
+
+    for i, ext in enumerate(_ad):
+
+        data = np.ma.masked_array(ext.data, mask=ext.mask)
+        cmap = plt.get_cmap('viridis')
+        cmap.set_bad('red', alpha='0.5')
+        axs[i].imshow(data, origin='lower', cmap=cmap, vmin=vmin, vmax=vmax)
+        # axs[i].imshow(data.data, origin='lower', vmin=vmin, vmax=vmax)
+
+    plt.show()
+
+
+# -- Recipe to create pre-processed data ---------------------------------------
+def create_master_bias_for_tests():
+    """
+    Creates input bias data for tests.
+
+    The raw files will be downloaded and saved inside the path stored in the
+    `$DRAGONS_TEST/raw_inputs` directory.
+
+    Processed files will be stored inside a new folder called
+    "./dragons_test_inputs". The sub-directory structure should reflect the one
+    returned by the `path_to_inputs` fixture.
+    """
+    bias_datasets = [
+        [f"N20190101S{n:04d}.fits" for n in range(494, 499)],
+    ]
+
+    root_path = os.path.join("./dragons_test_inputs/")
+    module_path = "geminidr/gmos/recipes/qa/{:s}/inputs".format(__file__.split('.')[0])
+    path = os.path.join(root_path, module_path)
+    os.makedirs(path, exist_ok=True)
+    os.chdir(path)
+
+    print('\n  Current working directory:\n    {:s}'.format(os.getcwd()))
+
+    for filenames in bias_datasets:
+
+        print('  Downloading files...')
+        paths = [download_from_archive(f) for f in filenames]
+
+        f = paths[0]
+        ad = astrodata.open(f)
+
+        if not os.path.exists(f.replace('.fits', '_bias.fits')):
+            print(f"  Creating input file:\n")
+            print(f"    {os.path.basename(f).replace('.fits', '_bias.fits')}")
+            logutils.config(file_name=f'log_arc_{ad.data_label()}.txt')
+            r = Reduce()
+            r.files.extend(paths)
+            r.runr()
+        else:
+            print(f"  Input file already exists:")
+            print(f"    {f.replace('.fits', '_bias.fits')}")
+
+
+if __name__ == '__main__':
+    from sys import argv
+
+    if '--create-inputs' in argv[1:]:
+        create_master_bias_for_tests()
+    else:
+        pytest.main()

geminidr/gmos/recipes/ql/tests/test_make_processed_flat.py → geminidr/gmos/recipes/ql/tests/test_flat_ls_spect.py

@@ -144,7 +144,7 @@ def create_master_bias_for_tests():
     should reflect the one returned by the `path_to_inputs` fixture.
     """
     root_path = os.path.join("./dragons_test_inputs/")
-    module_path = "geminidr/gmos/recipes/ql/test_make_processed_flat/"
+    module_path = f"geminidr/gmos/recipes/ql/{__file__.split('.')[0]}/"
     path = os.path.join(root_path, module_path, "inputs/")
 
     os.makedirs(path, exist_ok=True)