Merge branch 'release-v0.8.x' into stable

# Conflicts:
#	doc/conf.py
#	megaradrp/__init__.py
#	setup.py

README.rst

@@ -28,7 +28,7 @@ installed in order to be able to be installed and work properly:
  - numpy >= 1.7 (http://www.numpy.org/)
  - scipy (http://www.scipy.org)
  - astropy >= 2.0 (http://www.astropy.org/)
- - numina >= 0.16 (http://guaix.fis.ucm.es/projects/numina/)
+ - numina >= 0.17 (http://guaix.fis.ucm.es/projects/numina/)
  - scikit-image (http://scikit-image.org/)
 
 Webpage: https://guaix.fis.ucm.es/megara

doc/conf.py

@@ -27,8 +27,8 @@
 
 project = u'MEGARA Data Reduction Pipeline'
 copyright = u'2013-2018, Universidad Complutense de Madrid'
-version = '0.7'
-release = '0.7.1'
+version = '0.8'
+release = '0.8'
 show_authors = True
 
 numpydoc_show_class_members = False

doc/installation.rst

@@ -9,12 +9,12 @@ Requirements
 The MEGARA Pipeline package requires the following packages installed in order to
 be able to be installed and work properly:
 
- - `python 2.7 <https://www.python.org>`_
+ - `python <https://www.python.org>`_ either 2.7 or >= 3.4
  - `setuptools <http://peak.telecommunity.com/DevCenter/setuptools>`_
  - `numpy <http://www.numpy.org/>`_ >= 1.7
  - `scipy <http://www.scipy.org/>`_
- - `astropy <http://www.astropy.org/>`_ >= 1.1
- - `numina <https://pypi.python.org/pypi/numina/>`_ >= 0.15
+ - `astropy <http://www.astropy.org/>`_ >= 2.0
+ - `numina <https://pypi.python.org/pypi/numina/>`_ >= 0.17
  - `scikit-image <http://scikit-image.org/>`_
 
 Additional packages are optionally required:

megaradrp/__init__.py

@@ -12,7 +12,7 @@
 import logging
 
 
-__version__ = '0.7.1'
+__version__ = '0.8'
 
 
 # Top level NullHandler

megaradrp/datamodel.py

@@ -125,27 +125,30 @@ def get_imgid(self, img):
             return super(MegaraDataModel, self).get_imgid(img)
 
     def get_fiberconf(self, img):
-        """Obtain FIBER extension"""
+        """Obtain FiberConf from image"""
         main_insmode = img[0].header.get('INSMODE', '')
         if 'FIBERS' in img:
             # We have a 'fibers' extension
             # Information os there
             hdr_fiber = img['FIBERS'].header
             return read_fibers_extension(hdr_fiber, insmode=main_insmode)
         else:
-            insmode = img[0].header.get('INSMODE')
-            if insmode == 'LCB':
-                slit_file = 'lcb_default_header.txt'
-            elif insmode == 'MOS':
-                slit_file = 'mos_default_header.txt'
-            else:
-                # Read fiber info from headers
-                raise ValueError('Invalid INSMODE {}'.format(insmode))
-
-            data = pkgutil.get_data('megaradrp.instrument.configs', slit_file)
-            default_hdr = StringIO(data.decode('utf8'))
-            hdr_fiber = fits.header.Header.fromfile(default_hdr)
-            return read_fibers_extension(hdr_fiber)
+            return self.get_fiberconf_default(main_insmode)
+
+    def get_fiberconf_default(self, insmode):
+        """Obtain default FiberConf object"""
+        if insmode == 'LCB':
+            slit_file = 'lcb_default_header.txt'
+        elif insmode == 'MOS':
+            slit_file = 'mos_default_header.txt'
+        else:
+            # Read fiber info from headers
+            raise ValueError('Invalid INSMODE {}'.format(insmode))
+
+        data = pkgutil.get_data('megaradrp.instrument.configs', slit_file)
+        default_hdr = StringIO(data.decode('utf8'))
+        hdr_fiber = fits.header.Header.fromfile(default_hdr)
+        return read_fibers_extension(hdr_fiber)
 
     def gather_info_oresult(self, val):
         return [self.gather_info_dframe(f) for f in val.images]

megaradrp/drp.yaml

@@ -102,11 +102,6 @@ modes:
   description: Sensivity Star Recipe
   key: MegaraSensitivityStar
   tagger: megaradrp.taggers.tagger_base_image
-- name: Linearity test
-  summary: Test the linearity of the MEGARA detector
-  description: Test the linearity of the MEGARA detector
-  key: MegaraLinearirtyTest
-  tagger: megaradrp.taggers.tagger_base_image
 - name: Telescope Focus
   summary: Telescope Focus
   description: Telescope Focus
@@ -172,7 +167,6 @@ pipelines:
          keys: {key1: 1, key2: 2} 
          state: {}
       MegaraBadPixelMask: megaradrp.recipes.calibration.bpm.BadPixelsMaskRecipe
-      MegaraLinearityTest: megaradrp.recipes.calibration.linearity.LinearityTestRecipe
       MegaraBiasImage: megaradrp.recipes.calibration.bias.BiasRecipe
       MegaraDarkImage: megaradrp.recipes.calibration.dark.DarkRecipe
       MegaraArcCalibration: megaradrp.recipes.calibration.arc.ArcCalibrationRecipe

megaradrp/recipes/calibration/arc.py

@@ -418,6 +418,7 @@ def calibrate_wl(self, rss, lines_catalog, poldeg, tracemap, nlines,
 
         # save PDF file with plots in working directory
         if self.intermediate_results:
+            from numina.array.display.matplotlib_qt import plt
             from matplotlib.backends.backend_pdf import PdfPages
             pdf = PdfPages('wavecal_iter1.pdf')
             for dumplot in zip([plot_npeaksfound, plot_crval1, plot_cdelt1],
@@ -429,6 +430,7 @@ def calibrate_wl(self, rss, lines_catalog, poldeg, tracemap, nlines,
                                linestyle='', marker='.', color='C0',
                                show=False)
                 pdf.savefig()
+                plt.close()
             for ideg in range(poldeg_initial + 1):
                 dumplot = [coef[ideg] for coef in plot_coeff]
                 ax = ximplotxy(plot_tracenumber, dumplot,
@@ -437,6 +439,7 @@ def calibrate_wl(self, rss, lines_catalog, poldeg, tracemap, nlines,
                                linestyle='', marker='.', color='C0',
                                show=False)
                 pdf.savefig()
+                plt.close()
             pdf.close()
 
         self.logger.info('Generating fwhm_image...')
@@ -513,6 +516,8 @@ def calibrate_wl(self, rss, lines_catalog, poldeg, tracemap, nlines,
                                               ylogscale=True,
                                               pdf=pdf)
                     if pdf is not None:
+                        from numina.array.display.matplotlib_qt import plt
+                        plt.close()
                         pdf.close()
                     if poly_refined != numpy.polynomial.Polynomial([0.0]):
                         npoints_eff = yres_summary['npoints']
@@ -566,6 +571,7 @@ def calibrate_wl(self, rss, lines_catalog, poldeg, tracemap, nlines,
 
             # save PDF file with plots in working directory
             if self.intermediate_results:
+                from numina.array.display.matplotlib_qt import plt
                 from matplotlib.backends.backend_pdf import PdfPages
                 pdf = PdfPages('wavecal_iter2.pdf')
                 for dumplot in zip(
@@ -580,6 +586,7 @@ def calibrate_wl(self, rss, lines_catalog, poldeg, tracemap, nlines,
                                    linestyle='', marker='.', color='C0',
                                    show=False)
                     pdf.savefig()
+                    plt.close()
                 for ideg in range(poldeg_refined + 1):
                     dumplot = [coef[ideg] for coef in plot_coeff]
                     ax = ximplotxy(plot_tracenumber, dumplot,
@@ -588,6 +595,7 @@ def calibrate_wl(self, rss, lines_catalog, poldeg, tracemap, nlines,
                                    linestyle='', marker='.', color='C0',
                                    show=False)
                     pdf.savefig()
+                    plt.close()
                 pdf.close()
         else:
             data_wlcalib = None

megaradrp/recipes/calibration/flat.py

@@ -162,6 +162,7 @@ def obtain_fiber_flat(self, rss_wl, wlcalib, col1=1900, col2=2100, window=31, de
             plt.plot(xx, collapse_smooth, '-')
             plt.plot(xx, collapse_smooth_s, '--')
             plt.savefig('collapsed_smooth.png')
+            plt.close()
 
         # Divide each fiber in rss_wl by spectrum
         gmean = col_good_mean.mean()

megaradrp/requirements.py

@@ -106,5 +106,5 @@ def __init__(self, optional=True):
         super(ReferenceExtinction,
               self).__init__(megaradrp.types.ReferenceExtinctionTable,
                              "Reference extinction",
-                             optional=True
+                             optional=optional
                              )

megaradrp/tests/test_types.py

@@ -0,0 +1,76 @@
+
+import pytest
+
+import megaradrp.requirements as reqs
+import numina.core
+import numina.dal
+from numina.exceptions import NoResultFound
+
+
+class IoInstance(object):
+    _numina_desc_val = {}
+
+
+class LocalDal(object):
+    def __init__(self):
+        self.filename = "some_file.txt"
+
+    def search_product(self, name, tipo, obsres, options=None):
+        if obsres.id == 1:
+            return numina.dal.StoredProduct(1, self.filename, tags={})
+        if obsres.id == 2:
+            return numina.dal.StoredProduct(1, None, tags={})
+        elif obsres.id == 0:
+            raise NoResultFound('Nothing')
+        else:
+            sp = None
+        return sp
+
+
+@pytest.mark.parametrize('obsid', [0, 1, 2])
+def test_extinction_none(obsid):
+
+    name = "reference_extinction"
+    reference_extinction = reqs.ReferenceExtinction()
+    reference_extinction.dest = name
+
+    dal = LocalDal()
+
+    obs = numina.core.ObservationResult()
+    obs.id = obsid
+    obs.requirements = {name: None}
+    value = reference_extinction.query(dal, obs)
+    instance = IoInstance()
+
+    reference_extinction.__set__(instance, value)
+    cval = instance._numina_desc_val[name]
+    assert cval is None
+
+
+def _test_extinction_filename():
+
+    name = "reference_extinction"
+    reference_extinction = reqs.ReferenceExtinction()
+    reference_extinction.dest = name
+
+    dal = LocalDal()
+
+    obs = numina.core.ObservationResult()
+
+    import tempfile, os, numpy
+
+    arr = numpy.array([1.0,2.0,3.0,4.0])
+    fd, path = tempfile.mkstemp()
+    try:
+        #obs.requirements = {name: path}
+        numpy.savetxt(path, arr)
+
+        value = reference_extinction.query(dal, obs)
+        instance = IoInstance()
+
+        reference_extinction.__set__(instance, value)
+        cval = instance._numina_desc_val[name]
+        assert numpy.allclose(cval, arr)
+        assert False
+    finally:
+        os.remove(path)

megaradrp/types.py

@@ -11,7 +11,7 @@
 
 
 from numina.core import DataFrameType
-from numina.types.product import DataProductTag
+from numina.types.product import DataProductMixin
 from numina.types.datatype import DataType
 from numina.types.array import ArrayType
 from numina.types.linescatalog import LinesCatalog
@@ -55,15 +55,15 @@ class ProcessedSpectrum(ProcessedFrame):
     pass
 
 
-class ProcessedImageProduct(DataProductTag, ProcessedImage):
+class ProcessedImageProduct(DataProductMixin, ProcessedImage):
     pass
 
 
-class ProcessedRSSProduct(DataProductTag, ProcessedRSS):
+class ProcessedRSSProduct(DataProductMixin, ProcessedRSS):
     pass
 
 
-class ProcessedSpectrumProduct(DataProductTag, ProcessedSpectrum):
+class ProcessedSpectrumProduct(DataProductMixin, ProcessedSpectrum):
     pass
 
 
@@ -117,12 +117,17 @@ class MasterSensitivity(ProcessedSpectrumProduct):
     pass
 
 
-class ReferenceExtinctionTable(DataProductTag, ArrayType):
+class ReferenceExtinctionTable(DataProductMixin, ArrayType):
     """Atmospheric Extinction."""
-    pass
+    def convert(self, obj):
+        # Support None value
+        if obj is None:
+            return None
+        else:
+            return super(ReferenceExtinctionTable, self).convert(obj)
 
 
-class ReferenceSpectrumTable(DataProductTag, ArrayType):
+class ReferenceSpectrumTable(DataProductMixin, ArrayType):
     """The spectrum of a reference star"""
     pass