Merge 22547bb into da6c38b

docs/change_history.rst

@@ -3,9 +3,15 @@ Change History
 
 .. _v1.3.0:
 
-V1.3.0 Unreleased
+V1.3.0 06-03-2020
 ^^^^^^^^^^^^^^^^^
 
+- Made it compatible with Astropy 4.0
+- All versions are free except for Pandas [#314]
+- `wavelength.WavelengthCalibration.__call__` can now return a json output.
+- `core.setup_logging` can now create a generic logger (same format).
+- Modified how master bias are named.
+- Removed bias overscan and trimming correction on master bias creation.
 - Bugs Fixed:
 
   + `--max-targets` was not being used, missed connection in `MainApp`.
@@ -20,6 +26,7 @@ V1.3.0 Unreleased
   points, this allows to re use other master loggers.
 - Changed `--background-threshold` to multiply by detection limit instead of
   background level
+- Created standard JSON output for :class:`~wavelength.WavelengthCalibration`.
 
 
 .. _v1.2.1:

goodman_pipeline/core/core.py

@@ -2,24 +2,21 @@
                         unicode_literals)
 
 import calendar
+import ccdproc
 import collections
 import datetime
 import glob
 import logging
 import math
+import numpy as np
 import os
+import pandas
 import re
-import shutil
+import scipy
 import subprocess
 import sys
 import time
-from threading import Timer
-import pickle
 
-import ccdproc
-import numpy as np
-import pandas
-import scipy
 from astroplan import Observer
 from astropy import units as u
 from astropy.io import fits
@@ -30,9 +27,9 @@
 from astropy.time import Time
 from astroscrappy import detect_cosmics
 from ccdproc import CCDData, ImageFileCollection
-# matplotlib.use('Qt5Agg')
 from matplotlib import pyplot as plt
 from scipy import signal
+from threading import Timer
 
 from . import check_version
 
@@ -41,9 +38,6 @@
 log = logging.getLogger(__name__)
 
 
-
-
-
 def astroscrappy_lacosmic(ccd, red_path=None, save_mask=False):
 
     mask, ccd.data = detect_cosmics(ccd.data)
@@ -424,7 +418,7 @@ def create_master_flats(flat_files,
                         trim_section,
                         master_bias_name,
                         new_master_flat_name,
-                        saturation,
+                        saturation_threshold,
                         ignore_bias=False):
     """Creates master flats
 
@@ -450,7 +444,7 @@ def create_master_flats(flat_files,
         the full path as `raw_path` + `basename`.
         new_master_flat_name (str): Name of the file to save new master
         flat. Can be absolute path or not.
-        saturation (int): Saturation threshold, defines the percentage of
+        saturation_threshold (int): Saturation threshold, defines the percentage of
         pixels above saturation level allowed for flat field images.
         ignore_bias (bool): Flag to create master bias without master bias.
 
@@ -508,11 +502,11 @@ def create_master_flats(flat_files,
                 'Master bias image')
 
         else:
-            log.error('Unknown observation technique: ' + technique)
+            log.warning('Ignoring bias on request')
         if is_file_saturated(ccd=ccd,
-                             threshold=saturation):
+                             threshold=saturation_threshold):
             log.warning('Removing saturated image {:s}. '
-                        'Use --saturation to change saturation '
+                        'Use --saturation_threshold to change saturation_threshold '
                         'level'.format(flat_file))
             continue
         else:
@@ -541,7 +535,7 @@ def create_master_flats(flat_files,
         return master_flat, master_flat_name
     else:
         log.error('Empty flat list. Check that they do not exceed the '
-                  'saturation limit.')
+                  'saturation_threshold limit.')
         return None, None
 
 
@@ -1074,9 +1068,9 @@ def extraction(ccd,
           2D spectrum
         target_trace (object): Instance of astropy.modeling.Model, a low order
           polynomial that defines the trace of the spectrum in the ccd object.
-        spatial_profile (Model): Instance of astropy.modeling.Model, a Gaussian
-          model previously fitted to the spatial profile of the 2D spectrum
-          contained in the ccd object.
+        spatial_profile (Model): Instance of :class:`~astropy.modeling.Model`,
+          a Gaussian model previously fitted to the spatial profile of the 2D
+          spectrum contained in the ccd object.
         extraction_name (str): Extraction type, can be `fractional` or
           `optimal` though the optimal extraction is not implemented yet.
 
@@ -1991,9 +1985,9 @@ def image_trim(ccd, trim_section, trim_type='trimsec', add_keyword=False):
                                       'Slit trim section, slit illuminated '
                                       'area only.')
         else:
-            log.warning('Unrecognized trim type')
+            log.warning('Unrecognized trim type: {}'.format(trim_type))
             ccd.header['GSP_TRIM'] = (trim_section,
-                                      'Image trimmed by unreckognized method: '
+                                      'Image trimmed by unrecognized method: '
                                       '{:s}'.format(trim_type))
     else:
         log.info("{:s} trim section is not "
@@ -2038,13 +2032,13 @@ def interpolate(spectrum, interpolation_size):
 def is_file_saturated(ccd, threshold):
     """Detects a saturated image
 
-    It counts the number of pixels above the saturation level, then finds
+    It counts the number of pixels above the saturation_threshold level, then finds
     which percentage they represents and if it is above the threshold it
     will return True. The percentage threshold can be set using the command
-    line argument ``--saturation``.
+    line argument ``--saturation_threshold``.
 
     Args:
-        ccd (CCDData): Image to be tested for saturation
+        ccd (CCDData): Image to be tested for saturation_threshold
         threshold (float): Percentage of saturated pixels allowed. Default 1.
 
     Returns:
@@ -2066,7 +2060,7 @@ def is_file_saturated(ccd, threshold):
     if saturated_percent >= float(threshold):
         log.warning(
             "The current image has more than {:.2f} percent "
-            "of pixels above saturation level".format(float(threshold)))
+            "of pixels above saturation_threshold level".format(float(threshold)))
         return True
     else:
         return False
@@ -3912,18 +3906,18 @@ class SaturationValues(object):
     """
 
     def __init__(self, ccd=None):
-        """Defines a :class:`~pandas.DataFrame` with saturation information
+        """Defines a :class:`~pandas.DataFrame` with saturation_threshold information
 
-        Both, Red and Blue cameras have tabulated saturation values depending
+        Both, Red and Blue cameras have tabulated saturation_threshold values depending
         on the readout configurations. It defines a :class:`~pandas.DataFrame`
         object.
 
         Notes:
             For the purposes of this documentation *50% full well* is the same
-            as ``saturation level`` though they are not the same thing.
+            as ``saturation_threshold level`` though they are not the same thing.
 
         Args:
-            ccd (CCDData): Image to be tested for saturation
+            ccd (CCDData): Image to be tested for saturation_threshold
 
         """
         self.log = logging.getLogger(__name__)
@@ -3964,7 +3958,7 @@ def saturation_value(self):
         """Saturation value in counts
 
         In fact the value it returns is the 50% of full potential well,
-        Some configurations reach digital saturation before 50% of full
+        Some configurations reach digital saturation_threshold before 50% of full
         potential well, they are specified in the last column:
         ``saturates_before``.
 
@@ -3979,13 +3973,13 @@ def saturation_value(self):
             return self.__saturation
 
     def get_saturation_value(self, ccd):
-        """Defines the saturation level
+        """Defines the saturation_threshold level
 
         Args:
-            ccd (CCDData): Image to be tested for saturation
+            ccd (CCDData): Image to be tested for saturation_threshold
 
         Returns:
-            The saturation value or None
+            The saturation_threshold value or None
 
         """
         hfw = self._sdf.half_full_well[
@@ -3994,12 +3988,12 @@ def get_saturation_value(self, ccd):
             (self._sdf.read_noise == ccd.header['RDNOISE'])]
 
         if hfw.empty:
-            self.log.critical('Unable to obtain saturation level')
+            self.log.critical('Unable to obtain saturation_threshold level')
             self.__saturation = None
             return None
         else:
             self.__saturation = float(hfw.to_string(index=False))
-            self.log.debug("Set saturation level as {:.0f}".format(
+            self.log.debug("Set saturation_threshold level as {:.0f}".format(
                 self.__saturation))
             return self.__saturation
 

goodman_pipeline/core/tests/test_core.py

@@ -609,7 +609,7 @@ def test_create_master_flats_no_bias(self):
             trim_section=self.trim_section,
             master_bias_name='master_bias.fits',
             new_master_flat_name=self.master_flat_name,
-            saturation=1,
+            saturation_threshold=1,
             ignore_bias=True)
 
         self.assertEqual(self.master_flat_name, os.path.basename(name))
@@ -629,7 +629,7 @@ def test_create_master_flats_with_bias(self):
                                           'master_bias.fits'),
             new_master_flat_name=os.path.join(self.reduced_data,
                                               self.master_flat_name),
-            saturation=1,
+            saturation_threshold=1,
             ignore_bias=False)
 
         self.assertEqual(self.master_flat_name, os.path.basename(name))
@@ -656,7 +656,7 @@ def test_create_master_flats_saturated_flats(self):
             trim_section=self.trim_section,
             master_bias_name='master_bias.fits',
             new_master_flat_name=self.master_flat_name,
-            saturation=1,
+            saturation_threshold=1,
             ignore_bias=False)
 
         self.assertNotEqual(self.flat_files[0], master.header['GSP_IC01'])
@@ -671,7 +671,7 @@ def test_create_master_flats_empty_list(self):
             trim_section=self.trim_section,
             master_bias_name='master_bias.fits',
             new_master_flat_name=self.master_flat_name,
-            saturation=1,
+            saturation_threshold=1,
             ignore_bias=False)
         self.assertIsNone(master)
         self.assertIsNone(name)

goodman_pipeline/images/goodman_ccd.py

@@ -131,12 +131,12 @@ def get_args(arguments=None):
                         default='./RED',
                         help="Path to reduced data.")
 
-    parser.add_argument('--saturation',
+    parser.add_argument('--saturation_threshold',
                         action='store',
                         default=1.,
                         dest='saturation_threshold',
                         metavar='<value>',
-                        help="Maximum percent of pixels above saturation "
+                        help="Maximum percent of pixels above saturation_threshold "
                              "threshold. Default 1 percent.")
 
     parser.add_argument('--version',

goodman_pipeline/images/image_processor.py

@@ -142,7 +142,7 @@ def __call__(self):
                             trim_section=self.trim_section,
                             master_bias_name=self.master_bias_name,
                             new_master_flat_name=master_flat_name,
-                            saturation=self.args.saturation_threshold,
+                            saturation_threshold=self.args.saturation_threshold,
                             ignore_bias=self.args.ignore_bias)
                     else:
                         log.debug('Process Data Group')
@@ -219,7 +219,7 @@ def process_spectroscopy_science(self, science_group, save_all=False):
                             trim_section=self.trim_section,
                             master_bias_name=self.master_bias_name,
                             new_master_flat_name=master_flat_name,
-                            saturation=self.args.saturation_threshold,
+                            saturation_threshold=self.args.saturation_threshold,
                             ignore_bias=self.args.ignore_bias)
             elif self.args.ignore_flats:
                 log.warning('Ignoring creation of Master Flat by request.')
@@ -494,7 +494,7 @@ def process_spectroscopy_science(self, science_group, save_all=False):
                             trim_section=self.trim_section,
                             master_bias_name=self.master_bias_name,
                             new_master_flat_name=master_flat_name,
-                            saturation=self.args.saturation_threshold)
+                            saturation_threshold=self.args.saturation_threshold)
         else:
             log.error('There is no valid datatype in this group')
 

goodman_pipeline/images/tests/test_image_processor.py

@@ -14,7 +14,7 @@
 class ImageProcessorTest(TestCase):
 
     def setUp(self):
-        arguments = ['--saturation', '1']
+        arguments = ['--saturation_threshold', '1']
         args = get_args(arguments=arguments)
         data_container = NightDataContainer(path='/fake',
                                             instrument='Red',

goodman_pipeline/spectroscopy/redspec.py

@@ -527,7 +527,7 @@ def _run(self,
                             self.log.error(error)
 
 
-if __name__ == '__main__':
+if __name__ == '__main__':  # pragma: no cover
     MAIN_APP = MainApp()
     try:
         MAIN_APP()