Merge pull request #425 from desihub/skyresid

Offline QA for SkySub Residuals

doc/changes.rst

@@ -6,13 +6,23 @@ desispec Change Log
 -------------------
 
 * Small fixes to desi_qa_prod and qa_prod
+* Removes a number of QL metrics from offline qa
 * Fixes integration tests for desisim newexp refactor
 * Removes spectra grouping by brick; nside=64 healpix grouping default
-* Add get_nights method to io.meta
-* Add search_for_framefile method to io.frame
-* Add desi_qa_frame script to generate frame QA
+* Add get_nights method to io.meta (#422)
+* Add search_for_framefile method to io.frame (#422)
+* Add desi_qa_frame script to generate frame QA (#424)
+* Add frame_meta to parameters (for slurping the Frame headers)
+* Add get_reduced_frames() method to io.meta
+* Modifies QA_Prod meta file output to be JSON
+* Add load_meta() method to QA_Exposure
+* Add time_series ploting to desi_qa_prod
+* Add several new plots for skysub residuals
+* Adds method to generate QA Table for Prod
+* Refactor of skysubresid script
 
 .. _`#422`: https://github.com/desihub/desispec/pull/422
+.. _`#424`: https://github.com/desihub/desispec/pull/424
 
 0.15.2 (2017-07-12)
 -------------------

doc/qa.rst

@@ -53,6 +53,63 @@ Generate the QA YAML file and figures::
 
     desi_qa_frame --frame_file=frame-r7-00000077.fits --make_plots
 
+desi_qa_skyresid
+++++++++++++++++
+
+This script examines sky subtraction resdiuals
+for an exposure, night or production.
+
+usage
+-----
+
+Here is the usage::
+
+    usage: desi_qa_skyresid [-h] [--reduxdir PATH] [--expid EXPID] [--night NIGHT]
+                        [--channels CHANNELS] [--prod] [--gauss]
+                        [--nights NIGHTS]
+
+    Generate QA on Sky Subtraction residuals [v1.2]
+
+    optional arguments:
+      -h, --help           show this help message and exit
+      --reduxdir PATH      Override default path ($DESI_SPECTRO_REDUX/$SPECPROD)
+                           to processed data.
+      --expid EXPID        Generate exposure plot on given exposure
+      --night NIGHT        Generate night plot on given night
+      --channels CHANNELS  List of channels to include
+      --prod               Results for full production run
+      --gauss              Expore Gaussianity for full production run
+      --nights NIGHTS      List of nights to limit prod plots
+
+
+Exposure
+--------
+
+Generate a plot of the sky subtraction residuals for an
+input Exposure ID. e.g. ::
+
+    desi_qa_sky --expid=123
+
+Production
+----------
+
+Generate a plot of the sky subtraction residuals for the
+Production.  If reduxdir is not provided, then the script
+will use the $SPECPROD and $DESI_SPECTRO_REDUX environemental
+variables.  Simply called::
+
+    desi_qa_sky --prod
+
+Gaussianity
+-----------
+
+Examine whether the residuals are distributed
+as Gaussian statistics.  Here is an example::
+
+
+    desi_qa_sky --gauss
+
+
 desi_qa_prod
 ++++++++++++
 
@@ -64,16 +121,16 @@ usage
 
 Here is the usage::
 
-    usage: desi_qa_prod [-h] --specprod_dir SPECPROD_DIR
-                        [--make_frameqa MAKE_FRAMEQA] [--slurp] [--remove]
-                        [--clobber] [--channel_hist CHANNEL_HIST]
+    usage: desi_qa_prod [-h] [--reduxdir REDUXDIR] [--make_frameqa MAKE_FRAMEQA]
+                        [--slurp] [--remove] [--clobber]
+                        [--channel_hist CHANNEL_HIST] [--time_series TIME_SERIES]
 
-    Generate/Analyze Production Level QA [v1.2]
+    Generate/Analyze Production Level QA [v1.3]
 
     optional arguments:
       -h, --help            show this help message and exit
-      --specprod_dir SPECPROD_DIR
-                            Path containing the exposures/directory to use
+      --reduxdir REDUXDIR   Override default path ($DESI_SPECTRO_REDUX/$SPECPROD)
+                            to processed data.
       --make_frameqa MAKE_FRAMEQA
                             Bitwise flag to control remaking the QA files (1) and
                             figures (2) for each frame in the production
@@ -82,8 +139,9 @@ Here is the usage::
       --clobber             clobber existing QA files?
       --channel_hist CHANNEL_HIST
                             Generate channel histogram(s)
-
-
+      --time_series TIME_SERIES
+                            Generate time series plot. Input is QATYPE-METRIC,
+                            e.g. SKYSUB-MED_RESID
 
 frameqa
 -------
@@ -106,3 +164,22 @@ YAML file of all the QA outputs::
 
     desi_qa_prod --slurp   # Collate all the QA YAML files
     desi_qa_prod --slurp --remove  # Collate and remove the individual files
+
+Channel Histograms
+------------------
+
+Using the --channel_hist flag, the script will generate a series
+of histogram plots on default metrics: FIBERFLAT: MAX_RMS,
+SKYSUB: MED_RESID, FLUXCALIB: MAX_ZP_OFF::
+
+    desi_qa_prod --channel_hist
+
+Time Series Plot
+----------------
+
+Using the --time_series input with a *qatype* and *metric* produces
+a Time Series plot of that metric for all nights/exposures/frames
+in the production, by channel, e.g.::
+
+    desi_qa_prod --time_series=SKYSUB-MED_RESID
+    desi_qa_prod --time_series=FLUXCALIB-ZP

py/desispec/data/params/README

@@ -0,0 +1,3 @@
+desispec_param.yml
+  frame_types: List of allowed flavors
+  frame_meta: List of keywords that may be parsed from Header (e.g. for QA)

py/desispec/data/params/desispec_param.yml

@@ -1 +1,2 @@
 frame_types: ['none','flat','arc','bias','science']
+frame_meta: ['DATE-OBS', 'CAMERA', 'NIGHT', 'EXPTIME', 'AIRMASS', 'FLAVOR']

py/desispec/io/__init__.py

@@ -23,7 +23,7 @@
 from .meta import (findfile, get_exposures, get_files, get_raw_files,
                    rawdata_root, specprod_root, validate_night,
                    get_pipe_plandir, get_pipe_rundir, get_pipe_scriptdir,
-                   get_pipe_logdir, get_pipe_faildir, get_nights)
+                   get_pipe_logdir, get_pipe_faildir, get_reduced_frames, get_nights)
 from .params import read_params
 from .qa import (read_qa_frame, read_qa_data, write_qa_frame, write_qa_brick,
                  load_qa_frame, write_qa_exposure, write_qa_prod)

py/desispec/io/meta.py

@@ -300,7 +300,37 @@ def get_exposures(night, raw=False, rawdata_dir=None, specprod_dir=None):
     return sorted(exposures)
 
 
-def get_nights(strip_path=True, rawdata_dir=None, specprod_dir=None):
+def get_reduced_frames(channels=['b','r','z'], nights=None, ftype='cframe'):
+    """ Loops through a production to find all reduced frames (default is cframes)
+    One can choose a subset of reduced frames by argument
+    Args:
+        channels: list, optional
+        nights: list, optional
+        ftype: str, optional
+
+    Returns:
+        all_frames: list for frame filenames
+
+    """
+    all_frames = []
+    # Nights
+    if nights is None:
+        nights = get_nights()
+    # Loop on night
+    for night in nights:
+        exposures = get_exposures(night)
+        for exposure in exposures:
+            frames_dict = get_files(filetype=ftype, night=night, expid=exposure)
+            # Restrict on channel
+            for key in frames_dict.keys():
+                for channel in channels:
+                    if channel in key:
+                        all_frames.append(frames_dict[key])
+    # Return
+    return all_frames
+
+
+def get_nights(strip_path=True, specprod_dir=None):
     """
     Args:
         strip_path:  bool, optional; Strip the path to the nights folders

py/desispec/io/qa.py

@@ -7,6 +7,7 @@
 from __future__ import print_function, absolute_import, division
 
 import os, yaml
+import json
 
 from desiutil.io import yamlify
 
@@ -165,6 +166,7 @@ def write_qa_exposure(outroot, qaexp, ret_dict=False):
     # Generate the dict
     odict = {qaexp.night: {qaexp.expid: {}}}
     odict[qaexp.night][qaexp.expid]['flavor'] = qaexp.flavor
+    odict[qaexp.night][qaexp.expid]['meta'] = qaexp.meta
     cameras = list(qaexp.data['frames'].keys())
     for camera in cameras:
         odict[qaexp.night][qaexp.expid][camera] = qaexp.data['frames'][camera]
@@ -191,15 +193,16 @@ def load_qa_prod(inroot):
         odict : dict
     """
     log=get_logger()
-    infile = inroot+'.yaml'
+    infile = inroot+'.json'
     log.info("Loading QA prod file: {:s}".format(infile))
     # Read
-    odict = read_qa_data(infile)
+    with open(infile, 'rt') as fh:
+        odict = json.load(fh)
     # Return
     return odict
 
 
-def write_qa_prod(outroot, qaprod):
+def write_qa_prod(outroot, qaprod, indent=True):
     """Write QA for a given production
 
     Args:
@@ -208,11 +211,12 @@ def write_qa_prod(outroot, qaprod):
         qa_prod : QA_Prod object
 
     Returns:
-        outfile or odict : str or dict
+        outfile: str
+          output filename
     """
     from desiutil.io import combine_dicts
     log=get_logger()
-    outfile = outroot+'.yaml'
+    outfile = outroot+'.json'
     outfile = makepath(outfile, 'qa')
 
     # Loop on exposures
@@ -221,10 +225,10 @@ def write_qa_prod(outroot, qaprod):
         # Get the exposure dict
         idict = write_qa_exposure('foo', qaexp, ret_dict=True)
         odict = combine_dicts(odict, idict)
-    ydict = yamlify(odict)
-    # Simple yaml
-    with open(outfile, 'w') as yamlf:
-        yamlf.write( yaml.dump(ydict))#, default_flow_style=True) )
+    ydict = yamlify(odict)  # This works well for JSON too
+    # Simple json
+    with open(outfile, 'wt') as fh:
+        json.dump(ydict, fh, indent=indent)
     log.info('Wrote QA_Prod file: {:s}'.format(outfile))
 
     return outfile

py/desispec/qa/qa_exposure.py

@@ -46,6 +46,7 @@ def __init__(self, expid, night, flavor, specprod_dir=None, in_data=None, **kwar
         self.night = night
         self.specprod_dir = specprod_dir
         self.flavor = flavor
+        self.meta = {}
 
         if in_data is None:
             self.data = dict(flavor=self.flavor, expid=self.expid,
@@ -85,6 +86,17 @@ def fluxcalib(self, outfil):
         # Figure
         qa_plots.exposure_fluxcalib(outfil, self.data)
 
+    def load_meta(self, frame_meta):
+        """ Load meta info from input Frame meta
+        Args:
+            frame_meta:
+        """
+        desi_params = read_params()
+        for key in desi_params['frame_meta']:
+            if key in ['CAMERA']:  # Frame specific
+                continue
+            self.meta[key] = frame_meta[key]
+
     def load_qa_data(self, remove=False):
         """ Load the QA data files for a given exposure (currently yaml)
         Args: