From ba9e693466969b4cce388ca76196c9529c75f89a Mon Sep 17 00:00:00 2001 From: profxj Date: Tue, 8 Aug 2017 19:27:22 -0700 Subject: [PATCH 01/10] starting this branch --- bin/desi_qa_frame | 16 ++++++++ py/desispec/qa/qa_frame.py | 72 ++++++++++++++++++++++++++++++++- py/desispec/scripts/qa_frame.py | 62 ++++++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 2 deletions(-) create mode 100755 bin/desi_qa_frame create mode 100644 py/desispec/scripts/qa_frame.py diff --git a/bin/desi_qa_frame b/bin/desi_qa_frame new file mode 100755 index 000000000..75a3b309d --- /dev/null +++ b/bin/desi_qa_frame @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# +# See top-level LICENSE.rst file for Copyright information +# +# -*- coding: utf-8 -*- + +""" +This script generates QA related to a production +""" + +import desispec.scripts.qa_frame as qa_frame + + +if __name__ == '__main__': + args = qa_frame.parse() + qa_frame.main(args) diff --git a/py/desispec/qa/qa_frame.py b/py/desispec/qa/qa_frame.py index 7ce49a988..e681e5bfb 100644 --- a/py/desispec/qa/qa_frame.py +++ b/py/desispec/qa/qa_frame.py @@ -4,8 +4,7 @@ from __future__ import print_function, absolute_import, division -import numpy as np -import os +import warnings from desiutil.log import get_logger from desispec.io import read_params @@ -194,3 +193,72 @@ def __repr__(self): """ return ('{:s}: night={:s}, expid={:d}, camera={:s}, flavor={:s}'.format( self.__class__.__name__, self.night, self.expid, self.camera, self.flavor)) + + +def qaframe_from_frame(night, frame_fil, specprod_dir=None, make_plots=False): + from desispec.io import read_frame + from desispec.io import meta + from desispec.io.qa import load_qa_frame, write_qa_frame + from desispec.io.fiberflat import read_fiberflat + from desispec.qa import qa_plots + from desispec.io.sky import read_sky + from desispec.io.fluxcalibration import read_flux_calibration + # Load frame + frame = read_frame(frame_fil) + frame_meta = frame.meta + camera = frame_meta['CAMERA'].strip() + expid = frame_meta['EXPID'].strip() + spectro = int(frame_meta['CAMERA'][-1]) + if frame_meta['FLAVOR'] in ['flat', 'arc']: + qatype = 'qa_calib' + else: + qatype = 'qa_data' + # Load + qafile = meta.findfile(qatype, night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) + qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR']) + # Flat QA + if frame.meta['FLAVOR'] in ['flat']: + fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid, + specprod_dir=specprod_dir) + fiberflat = read_fiberflat(fiberflat_fil) + qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=True) + if make_plots: + # Do it + qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) + qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat) + # SkySub QA + if qatype == 'qa_data': + sky_fil = meta.findfile('sky', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) + try: + skymodel = read_sky(sky_fil) + except FileNotFoundError: + warnings.warn("Sky file {:s} not found. Skipping..".format(sky_fil)) + else: + qaframe.run_qa('SKYSUB', (frame, skymodel)) + if make_plots: + qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=expid, + specprod_dir=specprod_dir) + qa_plots.frame_skyres(qafig, frame, skymodel, qaframe) + # FluxCalib QA + if qatype == 'qa_data': + # Standard stars + stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir, + spectrograph=spectro) + # try: + # model_tuple=read_stdstar_models(stdstar_fil) + # except FileNotFoundError: + # warnings.warn("Standard star file {:s} not found. Skipping..".format(stdstar_fil)) + # else: + flux_fil = meta.findfile('calib', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir) + try: + fluxcalib = read_flux_calibration(flux_fil) + except FileNotFoundError: + warnings.warn("Flux file {:s} not found. Skipping..".format(flux_fil)) + else: + qaframe.run_qa('FLUXCALIB', (frame, fluxcalib)) # , model_tuple))#, indiv_stars)) + if make_plots: + qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=expid, + specprod_dir=specprod_dir) + qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib) # , model_tuple) + # Write + write_qa_frame(qafile, qaframe) diff --git a/py/desispec/scripts/qa_frame.py b/py/desispec/scripts/qa_frame.py new file mode 100644 index 000000000..9df3251b4 --- /dev/null +++ b/py/desispec/scripts/qa_frame.py @@ -0,0 +1,62 @@ +# Script for generating QA for a single Frame +from __future__ import absolute_import, division + +from desispec.qa import QA_Prod +from desiutil.log import get_logger +import argparse +import numpy as np + + +def parse(options=None): + parser = argparse.ArgumentParser(description="Generate Production Level QA") + parser.add_argument('--specprod_dir', type = str, default = None, required=False, + help = 'Path containing the exposures/directory to use.') + + + args = None + if options is None: + args = parser.parse_args() + else: + args = parser.parse_args(options) + return args + + +def main(args) : + + log=get_logger() + + log.info("starting") + + qa_prod = QA_Prod(args.specprod_dir) + + # Remake Frame QA? + if args.make_frameqa > 0: + log.info("(re)generating QA related to frames") + if (args.make_frameqa % 4) >= 2: + make_frame_plots = True + else: + make_frame_plots = False + # Run + qa_prod.make_frameqa(make_plots=make_frame_plots, clobber=args.clobber) + + # Slurp? + if args.slurp: + qa_prod.slurp(make=(args.make_frameqa > 0), remove=args.remove) + + # Channel histograms + if args.channel_hist is not None: + # imports + from matplotlib.backends.backend_pdf import PdfPages + from desispec.qa import qa_plots as dqqp + # + qa_prod.load_data() + outfile = qa_prod.prod_name+'_chist.pdf' + pp = PdfPages(outfile) + # Default? + if args.channel_hist == 'default': + dqqp.prod_channel_hist(qa_prod, 'FIBERFLAT', 'MAX_RMS', pp=pp, close=False) + dqqp.prod_channel_hist(qa_prod, 'SKYSUB', 'MED_RESID', xlim=(-1,1), pp=pp, close=False) + dqqp.prod_channel_hist(qa_prod, 'FLUXCALIB', 'MAX_ZP_OFF', pp=pp, close=False) + # Finish + print("Writing {:s}".format(outfile)) + pp.close() From 03b239dd66c33c731b1110f7640b94218c65c3dd Mon Sep 17 00:00:00 2001 From: profxj Date: Tue, 8 Aug 2017 20:00:04 -0700 Subject: [PATCH 02/10] working --- py/desispec/qa/qa_frame.py | 6 ++-- py/desispec/qa/qa_plots.py | 2 ++ py/desispec/scripts/qa_frame.py | 55 ++++++++++++--------------------- 3 files changed, 25 insertions(+), 38 deletions(-) diff --git a/py/desispec/qa/qa_frame.py b/py/desispec/qa/qa_frame.py index e681e5bfb..5ba33a30c 100644 --- a/py/desispec/qa/qa_frame.py +++ b/py/desispec/qa/qa_frame.py @@ -195,7 +195,7 @@ def __repr__(self): self.__class__.__name__, self.night, self.expid, self.camera, self.flavor)) -def qaframe_from_frame(night, frame_fil, specprod_dir=None, make_plots=False): +def qaframe_from_frame(night, frame_file, specprod_dir=None, make_plots=False): from desispec.io import read_frame from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame @@ -204,10 +204,10 @@ def qaframe_from_frame(night, frame_fil, specprod_dir=None, make_plots=False): from desispec.io.sky import read_sky from desispec.io.fluxcalibration import read_flux_calibration # Load frame - frame = read_frame(frame_fil) + frame = read_frame(frame_file) frame_meta = frame.meta camera = frame_meta['CAMERA'].strip() - expid = frame_meta['EXPID'].strip() + expid = frame_meta['EXPID'] spectro = int(frame_meta['CAMERA'][-1]) if frame_meta['FLAVOR'] in ['flat', 'arc']: qatype = 'qa_calib' diff --git a/py/desispec/qa/qa_plots.py b/py/desispec/qa/qa_plots.py index a35e5aa79..df7a578fa 100644 --- a/py/desispec/qa/qa_plots.py +++ b/py/desispec/qa/qa_plots.py @@ -97,6 +97,7 @@ def frame_skyres(outfil, frame, skymodel, qaframe, quick_look=False): skymodel: SkyModel object qaframe: QAFrame object """ + from desispec.sky import subtract_sky # Access metrics ''' @@ -108,6 +109,7 @@ def frame_skyres(outfil, frame, skymodel, qaframe, quick_look=False): pchi2_med = scipy.stats.chisqprob(chi2_med, dof_wavg) ''' skyfibers = np.array(qaframe.qa_data['SKYSUB']["METRICS"]["SKY_FIBERID"]) + subtract_sky(frame, skymodel) res=frame.flux[skyfibers] res_ivar=frame.ivar[skyfibers] if quick_look: diff --git a/py/desispec/scripts/qa_frame.py b/py/desispec/scripts/qa_frame.py index 9df3251b4..30b3ff845 100644 --- a/py/desispec/scripts/qa_frame.py +++ b/py/desispec/scripts/qa_frame.py @@ -9,8 +9,14 @@ def parse(options=None): parser = argparse.ArgumentParser(description="Generate Production Level QA") - parser.add_argument('--specprod_dir', type = str, default = None, required=False, - help = 'Path containing the exposures/directory to use.') + parser.add_argument('--frame_file', type = str, required=True, + help='Frame filename including path as needed') + parser.add_argument('--night', type = str, required=True, + help='Night of the exposure') + parser.add_argument('--reduxdir', type = str, default = None, metavar = 'PATH', + help = 'Override default path ($DESI_SPECTRO_REDUX/$SPECPROD) to processed data.') + parser.add_argument('--make_plots', default=False, action="store_true", + help = 'Generate QA figs too?') args = None @@ -23,40 +29,19 @@ def parse(options=None): def main(args) : + from desispec.io import meta + from desispec.io import read_meta_frame + from desispec.qa.qa_frame import qaframe_from_frame log=get_logger() log.info("starting") + if args.reduxdir is None: + specprod_dir = meta.specprod_root() + else: + specprod_dir = args.reduxdir + + # Generate qaframe (and figures?) + qaframe_from_frame(args.night, args.frame_file, + specprod_dir=specprod_dir, make_plots=args.make_plots) + - qa_prod = QA_Prod(args.specprod_dir) - - # Remake Frame QA? - if args.make_frameqa > 0: - log.info("(re)generating QA related to frames") - if (args.make_frameqa % 4) >= 2: - make_frame_plots = True - else: - make_frame_plots = False - # Run - qa_prod.make_frameqa(make_plots=make_frame_plots, clobber=args.clobber) - - # Slurp? - if args.slurp: - qa_prod.slurp(make=(args.make_frameqa > 0), remove=args.remove) - - # Channel histograms - if args.channel_hist is not None: - # imports - from matplotlib.backends.backend_pdf import PdfPages - from desispec.qa import qa_plots as dqqp - # - qa_prod.load_data() - outfile = qa_prod.prod_name+'_chist.pdf' - pp = PdfPages(outfile) - # Default? - if args.channel_hist == 'default': - dqqp.prod_channel_hist(qa_prod, 'FIBERFLAT', 'MAX_RMS', pp=pp, close=False) - dqqp.prod_channel_hist(qa_prod, 'SKYSUB', 'MED_RESID', xlim=(-1,1), pp=pp, close=False) - dqqp.prod_channel_hist(qa_prod, 'FLUXCALIB', 'MAX_ZP_OFF', pp=pp, close=False) - # Finish - print("Writing {:s}".format(outfile)) - pp.close() From ea157b2b3b36af1a29d837746e64ac892c23eb49 Mon Sep 17 00:00:00 2001 From: profxj Date: Tue, 8 Aug 2017 20:30:35 -0700 Subject: [PATCH 03/10] polishing --- py/desispec/scripts/qa_frame.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/desispec/scripts/qa_frame.py b/py/desispec/scripts/qa_frame.py index 30b3ff845..af3c8f9b0 100644 --- a/py/desispec/scripts/qa_frame.py +++ b/py/desispec/scripts/qa_frame.py @@ -41,7 +41,7 @@ def main(args) : specprod_dir = args.reduxdir # Generate qaframe (and figures?) - qaframe_from_frame(args.night, args.frame_file, + _ = qaframe_from_frame(args.night, args.frame_file, specprod_dir=specprod_dir, make_plots=args.make_plots) From 247c08478a4d725a354964e8887adfb364f56387 Mon Sep 17 00:00:00 2001 From: profxj Date: Tue, 8 Aug 2017 20:43:20 -0700 Subject: [PATCH 04/10] good to go (except a test) --- py/desispec/qa/qa_frame.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/py/desispec/qa/qa_frame.py b/py/desispec/qa/qa_frame.py index 5ba33a30c..376e6dc91 100644 --- a/py/desispec/qa/qa_frame.py +++ b/py/desispec/qa/qa_frame.py @@ -196,6 +196,17 @@ def __repr__(self): def qaframe_from_frame(night, frame_file, specprod_dir=None, make_plots=False): + """ Generate a qaframe object from an input frame_file name (and night) + Will also make plots if directed + Args: + night: str + frame_file: str + specprod_dir: str, optional + make_plots: bool, optional + + Returns: + + """ from desispec.io import read_frame from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame @@ -262,3 +273,4 @@ def qaframe_from_frame(night, frame_file, specprod_dir=None, make_plots=False): qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib) # , model_tuple) # Write write_qa_frame(qafile, qaframe) + return qaframe From 17f235074a0292d78bfa4b4db5bfb7bb63bd4bbd Mon Sep 17 00:00:00 2001 From: profxj Date: Thu, 10 Aug 2017 20:51:20 -0700 Subject: [PATCH 05/10] qa_frame advances --- doc/index.rst | 1 + doc/qa.rst | 74 +++++++++++++++++++++++++++++++++ py/desispec/io/__init__.py | 2 +- py/desispec/io/frame.py | 24 ++++++++++- py/desispec/io/meta.py | 14 +++++++ py/desispec/io/qa.py | 5 ++- py/desispec/qa/qa_frame.py | 13 ++++-- py/desispec/scripts/qa_frame.py | 12 ++---- py/desispec/scripts/qa_prod.py | 2 +- 9 files changed, 130 insertions(+), 17 deletions(-) create mode 100644 doc/qa.rst diff --git a/doc/index.rst b/doc/index.rst index 7fad5d66c..8e88e3609 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -13,6 +13,7 @@ Contents pipeline.rst dev.rst coadd.rst + qa.rst changes.rst api.rst diff --git a/doc/qa.rst b/doc/qa.rst new file mode 100644 index 000000000..85488ff68 --- /dev/null +++ b/doc/qa.rst @@ -0,0 +1,74 @@ +.. _qa: + +***************** +Quality Assurance +***************** + +Overview +================= + +The DESI spectroscopic pipeline includes a series of +routines that monitor the quality of the pipeline products +and may be used to inspect outputs across exposures, nights, +or a full production. + + +Scripts +======= + +desi_qa_frame ++++++++++++++ + +desi_qa_prod +++++++++++++ + +This script is used to both generate and analyze the +QA outputs for a complete production. + +usage +----- + +Here is the usage:: + + usage: desi_qa_prod [-h] --specprod_dir SPECPROD_DIR + [--make_frameqa MAKE_FRAMEQA] [--slurp] [--remove] + [--clobber] [--channel_hist CHANNEL_HIST] + + Generate/Analyze Production Level QA [v1.2] + + optional arguments: + -h, --help show this help message and exit + --specprod_dir SPECPROD_DIR + Path containing the exposures/directory to use + --make_frameqa MAKE_FRAMEQA + Bitwise flag to control remaking the QA files (1) and + figures (2) for each frame in the production + --slurp slurp production QA files into one? + --remove remove frame QA files? + --clobber clobber existing QA files? + --channel_hist CHANNEL_HIST + Generate channel histogram(s) + + + +frameqa +------- + +One generates the frame QA, the YAML and/or figure files +with the --make_frameqa flag:: + + desi_qa_prod --make_frameqa=1 # Generate all the QA YAML files + desi_qa_prod --make_frameqa=2 # Generate all the QA figure files + desi_qa_prod --make_frameqa=3 # Generate YAML and figures + +The optional --remove and --clobber flags can be used to remove/clobber +the QA files. + +slurp +----- + +By using the --slurp flag, one generates a full +YAML file of all the QA outputs:: + + desi_qa_prod --slurp # Collate all the QA YAML files + desi_qa_prod --slurp --remove # Collate and remove the individual files diff --git a/py/desispec/io/__init__.py b/py/desispec/io/__init__.py index 2f8982d9a..5867f43bb 100644 --- a/py/desispec/io/__init__.py +++ b/py/desispec/io/__init__.py @@ -23,7 +23,7 @@ from .meta import (findfile, get_exposures, get_files, get_raw_files, rawdata_root, specprod_root, validate_night, get_pipe_plandir, get_pipe_rundir, get_pipe_scriptdir, - get_pipe_logdir, get_pipe_faildir) + get_pipe_logdir, get_pipe_faildir, get_nights) from .params import read_params from .qa import (read_qa_frame, read_qa_data, write_qa_frame, write_qa_brick, load_qa_frame, write_qa_exposure, write_qa_prod) diff --git a/py/desispec/io/frame.py b/py/desispec/io/frame.py index 462abc6d3..8c0c3f5e6 100644 --- a/py/desispec/io/frame.py +++ b/py/desispec/io/frame.py @@ -15,11 +15,10 @@ from desiutil.io import encode_table from ..frame import Frame -from .meta import findfile +from .meta import findfile, get_nights, get_exposures from .util import fitsheader, native_endian, makepath from desiutil.log import get_logger - def write_frame(outfile, frame, header=None, fibermap=None, units=None): """Write a frame fits file and returns path to file written. @@ -169,3 +168,24 @@ def read_frame(filename, nspec=None): log.error("Frame did not pass simple vetting test. diagnosis={:d}".format(diagnosis)) # Return return frame + + +def search_for_framefile(frame_file): + log=get_logger() + # Parse frame file + ifile = frame_file.split('/')[-1] + splits = ifile.split('-') + root = splits[0] + camera = splits[1] + fexposure = int(splits[2].split('.')[0]) + + # Loop on nights + nights = get_nights() + for night in nights: + for exposure in get_exposures(night): + if exposure == fexposure: + mfile = findfile(root, camera=camera, night=night, expid=exposure) + if os.path.isfile(mfile): + return mfile + else: + log.error("Expected file {:s} not found..".format(mfile)) diff --git a/py/desispec/io/meta.py b/py/desispec/io/meta.py index a214cbe7b..cf4389f4a 100644 --- a/py/desispec/io/meta.py +++ b/py/desispec/io/meta.py @@ -300,6 +300,20 @@ def get_exposures(night, raw=False, rawdata_dir=None, specprod_dir=None): return sorted(exposures) +def get_nights(strip_path=True, rawdata_dir=None, specprod_dir=None): + # Init + if specprod_dir is None: + specprod_dir = specprod_root() + # Glob for nights + exp_path = os.path.join(specprod_dir,'exposures') + nights_path = glob.glob(exp_path+'/*') + # Strip off path? + if strip_path: + return [night_path.split('/')[-1] for night_path in nights_path] + else: + return nights_path + + def rawdata_root(): """Returns directory root for raw data, i.e. ``$DESI_SPECTRO_DATA`` diff --git a/py/desispec/io/qa.py b/py/desispec/io/qa.py index fdb82a89c..4df6e85f6 100644 --- a/py/desispec/io/qa.py +++ b/py/desispec/io/qa.py @@ -125,7 +125,7 @@ def write_qa_brick(outfile, qabrick): return outfile -def write_qa_frame(outfile, qaframe): +def write_qa_frame(outfile, qaframe, verbose=False): """Write QA for a given frame Args: @@ -134,6 +134,7 @@ def write_qa_frame(outfile, qaframe): qa_exp : QA_Frame object, with the following attributes qa_data: dict of QA info """ + log=get_logger() outfile = makepath(outfile, 'qa') # Generate the dict @@ -143,6 +144,8 @@ def write_qa_frame(outfile, qaframe): # Simple yaml with open(outfile, 'w') as yamlf: yamlf.write( yaml.dump(ydict))#, default_flow_style=True) ) + if verbose: + log.info("Wrote QA frame file: {:s}".format(outfile)) return outfile diff --git a/py/desispec/qa/qa_frame.py b/py/desispec/qa/qa_frame.py index 376e6dc91..19d48d057 100644 --- a/py/desispec/qa/qa_frame.py +++ b/py/desispec/qa/qa_frame.py @@ -195,11 +195,11 @@ def __repr__(self): self.__class__.__name__, self.night, self.expid, self.camera, self.flavor)) -def qaframe_from_frame(night, frame_file, specprod_dir=None, make_plots=False): +def qaframe_from_frame(frame_file, specprod_dir=None, make_plots=False): """ Generate a qaframe object from an input frame_file name (and night) + Write QA to disk Will also make plots if directed Args: - night: str frame_file: str specprod_dir: str, optional make_plots: bool, optional @@ -210,13 +210,20 @@ def qaframe_from_frame(night, frame_file, specprod_dir=None, make_plots=False): from desispec.io import read_frame from desispec.io import meta from desispec.io.qa import load_qa_frame, write_qa_frame + from desispec.io.frame import search_for_framefile from desispec.io.fiberflat import read_fiberflat from desispec.qa import qa_plots from desispec.io.sky import read_sky from desispec.io.fluxcalibration import read_flux_calibration + + if '/' in frame_file: # If present, assume full path is used here + pass + else: # Find the frame file in the desispec hierarchy? + frame_file = search_for_framefile(frame_file) # Load frame frame = read_frame(frame_file) frame_meta = frame.meta + night = frame_meta['NIGHT'].strip() camera = frame_meta['CAMERA'].strip() expid = frame_meta['EXPID'] spectro = int(frame_meta['CAMERA'][-1]) @@ -272,5 +279,5 @@ def qaframe_from_frame(night, frame_file, specprod_dir=None, make_plots=False): specprod_dir=specprod_dir) qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib) # , model_tuple) # Write - write_qa_frame(qafile, qaframe) + write_qa_frame(qafile, qaframe, verbose=True) return qaframe diff --git a/py/desispec/scripts/qa_frame.py b/py/desispec/scripts/qa_frame.py index af3c8f9b0..71c918acf 100644 --- a/py/desispec/scripts/qa_frame.py +++ b/py/desispec/scripts/qa_frame.py @@ -1,18 +1,14 @@ # Script for generating QA for a single Frame from __future__ import absolute_import, division -from desispec.qa import QA_Prod from desiutil.log import get_logger import argparse -import numpy as np def parse(options=None): - parser = argparse.ArgumentParser(description="Generate Production Level QA") + parser = argparse.ArgumentParser(description="Generate Frame Level QA [v1.0]") parser.add_argument('--frame_file', type = str, required=True, - help='Frame filename including path as needed') - parser.add_argument('--night', type = str, required=True, - help='Night of the exposure') + help='Frame filename. Full path is not required nor desired. ') parser.add_argument('--reduxdir', type = str, default = None, metavar = 'PATH', help = 'Override default path ($DESI_SPECTRO_REDUX/$SPECPROD) to processed data.') parser.add_argument('--make_plots', default=False, action="store_true", @@ -30,7 +26,6 @@ def parse(options=None): def main(args) : from desispec.io import meta - from desispec.io import read_meta_frame from desispec.qa.qa_frame import qaframe_from_frame log=get_logger() @@ -41,7 +36,6 @@ def main(args) : specprod_dir = args.reduxdir # Generate qaframe (and figures?) - _ = qaframe_from_frame(args.night, args.frame_file, - specprod_dir=specprod_dir, make_plots=args.make_plots) + _ = qaframe_from_frame(args.frame_file, specprod_dir=specprod_dir, make_plots=args.make_plots) diff --git a/py/desispec/scripts/qa_prod.py b/py/desispec/scripts/qa_prod.py index c1a0e4e8b..61c3b8b98 100644 --- a/py/desispec/scripts/qa_prod.py +++ b/py/desispec/scripts/qa_prod.py @@ -8,7 +8,7 @@ def parse(options=None): - parser = argparse.ArgumentParser(description="Generate Production Level QA") + parser = argparse.ArgumentParser(description="Generate/Analyze Production Level QA [v1.2]") parser.add_argument('--specprod_dir', type = str, default = None, required=True, help = 'Path containing the exposures/directory to use') From 02d2b7c26deaef1381a34880cd8a9ed18dadff3f Mon Sep 17 00:00:00 2001 From: profxj Date: Thu, 10 Aug 2017 20:53:30 -0700 Subject: [PATCH 06/10] more docs --- doc/qa.rst | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/doc/qa.rst b/doc/qa.rst index 85488ff68..44f3cd9b5 100644 --- a/doc/qa.rst +++ b/doc/qa.rst @@ -19,6 +19,37 @@ Scripts desi_qa_frame +++++++++++++ +Generate the QA for an input frame file. +The code can be written anywhere and the +output is written to its `proper' location. + +usage +----- + +Here is the usage:: + + usage: desi_qa_frame [-h] --frame_file FRAME_FILE [--reduxdir PATH] + [--make_plots] + + Generate Frame Level QA [v1.0] + + optional arguments: + -h, --help show this help message and exit + --frame_file FRAME_FILE + Frame filename. Full path is not required nor desired. + --reduxdir PATH Override default path ($DESI_SPECTRO_REDUX/$SPECPROD) + to processed data. + --make_plots Generate QA figs too? + + +examples +-------- + +Generate the QA YAML file:: + + desi_qa_frame --frame_file=frame-r7-00000077.fits + desi_qa_frame --frame_file=frame-r7-00000077.fits --make_plots + desi_qa_prod ++++++++++++ From 3eee0b92c99dc4294abd8b8ed833a42333753244 Mon Sep 17 00:00:00 2001 From: profxj Date: Thu, 10 Aug 2017 21:26:49 -0700 Subject: [PATCH 07/10] tests --- py/desispec/test/test_io.py | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/py/desispec/test/test_io.py b/py/desispec/test/test_io.py index a2a5e2692..589d482d8 100644 --- a/py/desispec/test/test_io.py +++ b/py/desispec/test/test_io.py @@ -583,6 +583,45 @@ def test_findfile_outdir(self): x = findfile('fibermap', night='20150101', expid=123, outdir=outdir) self.assertEqual(x, os.path.join(outdir, os.path.basename(x))) + def test_get_nights(self): + """ Test desispec.io.meta.get_nights + """ + from ..io.meta import get_nights + from ..io.meta import findfile + from ..io.util import makepath + os.environ['DESI_SPECTRO_REDUX'] = self.testEnv['DESI_SPECTRO_REDUX'] + os.environ['SPECPROD'] = self.testEnv['SPECPROD'] + # Generate dummy path + for night in ['20150101', '20150102']: + x = findfile('frame', camera='b0', night=night, expid=123) + makepath(x) + # Search for nights + nights = get_nights() + self.assertEqual(len(nights), 2) + self.assertTrue(isinstance(nights, list)) + self.assertTrue('20150102' in nights) + # Keep path + nights = get_nights(strip_path=False) + self.assertTrue('/' in nights[0]) + + def test_search_framefile(self): + """ Test desispec.io.frame.search_for_framefile + """ + from ..io.frame import search_for_framefile + from ..io.meta import findfile + from ..io.util import makepath + # Setup paths + os.environ['DESI_SPECTRO_REDUX'] = self.testEnv['DESI_SPECTRO_REDUX'] + os.environ['SPECPROD'] = self.testEnv['SPECPROD'] + # Generate a dummy frame file + x = findfile('frame', camera='b0', night='20150101', expid=123) + makepath(x) + with open(x,'a') as f: + pass + # Find it + mfile = search_for_framefile('frame-b0-000123.fits') + self.assertEqual(x, mfile) + @unittest.skipUnless(os.path.exists(os.path.join(os.environ['HOME'],'.netrc')),"No ~/.netrc file detected.") def test_download(self): """Test desiutil.io.download. From eaf6c7843d053bf756bd0ed17b66de9ab34b1a2e Mon Sep 17 00:00:00 2001 From: profxj Date: Thu, 10 Aug 2017 21:37:14 -0700 Subject: [PATCH 08/10] changes and qa.rst fix --- doc/changes.rst | 5 +++++ doc/qa.rst | 7 +++++-- py/desispec/io/frame.py | 10 +++++++++- py/desispec/io/meta.py | 9 +++++++++ 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/doc/changes.rst b/doc/changes.rst index ef249c772..b0caef15c 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -8,6 +8,11 @@ desispec Change Log * Small fixes to desi_qa_prod and qa_prod * Fixes integration tests for desisim newexp refactor * Removes spectra grouping by brick; nside=64 healpix grouping default +* Add get_nights method to io.meta +* Add search_for_framefile method to io.frame +* Add desi_qa_frame script to generate frame QA + +.. _`#422`: https://github.com/desihub/desispec/pull/422 0.15.2 (2017-07-12) ------------------- diff --git a/doc/qa.rst b/doc/qa.rst index 44f3cd9b5..8a073689a 100644 --- a/doc/qa.rst +++ b/doc/qa.rst @@ -5,7 +5,7 @@ Quality Assurance ***************** Overview -================= +======== The DESI spectroscopic pipeline includes a series of routines that monitor the quality of the pipeline products @@ -21,7 +21,7 @@ desi_qa_frame Generate the QA for an input frame file. The code can be written anywhere and the -output is written to its `proper' location. +output is written to its "proper" location. usage ----- @@ -48,6 +48,9 @@ examples Generate the QA YAML file:: desi_qa_frame --frame_file=frame-r7-00000077.fits + +Generate the QA YAML file and figures:: + desi_qa_frame --frame_file=frame-r7-00000077.fits --make_plots desi_qa_prod diff --git a/py/desispec/io/frame.py b/py/desispec/io/frame.py index 8c0c3f5e6..66e258af8 100644 --- a/py/desispec/io/frame.py +++ b/py/desispec/io/frame.py @@ -171,8 +171,16 @@ def read_frame(filename, nspec=None): def search_for_framefile(frame_file): + """ Search for an input frame_file in the desispec redux hierarchy + Args: + frame_file: str + + Returns: + mfile: str, full path to frame_file if found else raise error + + """ log=get_logger() - # Parse frame file + # Parse frame file name ifile = frame_file.split('/')[-1] splits = ifile.split('-') root = splits[0] diff --git a/py/desispec/io/meta.py b/py/desispec/io/meta.py index cf4389f4a..a80835166 100644 --- a/py/desispec/io/meta.py +++ b/py/desispec/io/meta.py @@ -301,6 +301,15 @@ def get_exposures(night, raw=False, rawdata_dir=None, specprod_dir=None): def get_nights(strip_path=True, rawdata_dir=None, specprod_dir=None): + """ + Args: + strip_path: bool, optional; Strip the path to the nights folders + rawdata_dir: + specprod_dir: + + Returns: + nights: list of nights (without or with paths) + """ # Init if specprod_dir is None: specprod_dir = specprod_root() From 9492c4e49af54a8ec8591af6cabb70b47a9f5023 Mon Sep 17 00:00:00 2001 From: profxj Date: Fri, 11 Aug 2017 21:19:01 -0700 Subject: [PATCH 09/10] edits to SB --- py/desispec/io/frame.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/py/desispec/io/frame.py b/py/desispec/io/frame.py index 66e258af8..1327eeda0 100644 --- a/py/desispec/io/frame.py +++ b/py/desispec/io/frame.py @@ -180,8 +180,8 @@ def search_for_framefile(frame_file): """ log=get_logger() - # Parse frame file name - ifile = frame_file.split('/')[-1] + # Parse frame file + path, ifile = os.path.split(frame_file) splits = ifile.split('-') root = splits[0] camera = splits[1] From 03010aa56a3062a36b63c97c2ee12555a5584d54 Mon Sep 17 00:00:00 2001 From: profxj Date: Sun, 13 Aug 2017 18:45:25 -0700 Subject: [PATCH 10/10] reduce frame windows --- py/desispec/qa/qa_plots.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/py/desispec/qa/qa_plots.py b/py/desispec/qa/qa_plots.py index df7a578fa..60d1c0068 100644 --- a/py/desispec/qa/qa_plots.py +++ b/py/desispec/qa/qa_plots.py @@ -121,7 +121,10 @@ def frame_skyres(outfil, frame, skymodel, qaframe, quick_look=False): # Plot fig = plt.figure(figsize=(8, 10.0)) - gs = gridspec.GridSpec(4,2) + if quick_look: + gs = gridspec.GridSpec(4,2) + else: + gs = gridspec.GridSpec(2,2) xmin,xmax = np.min(frame.wave), np.max(frame.wave) # Simple residual plot