diff --git a/doc/qa.rst b/doc/qa.rst index 8a073689a..fcfb7f99f 100644 --- a/doc/qa.rst +++ b/doc/qa.rst @@ -53,6 +53,63 @@ Generate the QA YAML file and figures:: desi_qa_frame --frame_file=frame-r7-00000077.fits --make_plots +desi_qa_skyresid +++++++++++++++++ + +This script examines sky subtraction resdiuals +for an exposure, night or production. + +usage +----- + +Here is the usage:: + + usage: desi_qa_skyresid [-h] [--reduxdir PATH] [--expid EXPID] [--night NIGHT] + [--channels CHANNELS] [--prod] [--gauss] + [--nights NIGHTS] + + Generate QA on Sky Subtraction residuals [v1.2] + + optional arguments: + -h, --help show this help message and exit + --reduxdir PATH Override default path ($DESI_SPECTRO_REDUX/$SPECPROD) + to processed data. + --expid EXPID Generate exposure plot on given exposure + --night NIGHT Generate night plot on given night + --channels CHANNELS List of channels to include + --prod Results for full production run + --gauss Expore Gaussianity for full production run + --nights NIGHTS List of nights to limit prod plots + + +Exposure +-------- + +Generate a plot of the sky subtraction residuals for an +input Exposure ID. e.g. :: + + desi_qa_sky --expid=123 + +Production +---------- + +Generate a plot of the sky subtraction residuals for the +Production. If reduxdir is not provided, then the script +will use the $SPECPROD and $DESI_SPECTRO_REDUX environemental +variables. Simply called:: + + desi_qa_sky --prod + +Gaussianity +----------- + +Examine whether the residuals are distributed +as Gaussian statistics. Here is an example:: + + + desi_qa_sky --gauss + + desi_qa_prod ++++++++++++ @@ -64,16 +121,16 @@ usage Here is the usage:: - usage: desi_qa_prod [-h] --specprod_dir SPECPROD_DIR - [--make_frameqa MAKE_FRAMEQA] [--slurp] [--remove] - [--clobber] [--channel_hist CHANNEL_HIST] + usage: desi_qa_prod [-h] [--reduxdir REDUXDIR] [--make_frameqa MAKE_FRAMEQA] + [--slurp] [--remove] [--clobber] + [--channel_hist CHANNEL_HIST] [--time_series TIME_SERIES] - Generate/Analyze Production Level QA [v1.2] + Generate/Analyze Production Level QA [v1.3] optional arguments: -h, --help show this help message and exit - --specprod_dir SPECPROD_DIR - Path containing the exposures/directory to use + --reduxdir REDUXDIR Override default path ($DESI_SPECTRO_REDUX/$SPECPROD) + to processed data. --make_frameqa MAKE_FRAMEQA Bitwise flag to control remaking the QA files (1) and figures (2) for each frame in the production @@ -82,8 +139,9 @@ Here is the usage:: --clobber clobber existing QA files? --channel_hist CHANNEL_HIST Generate channel histogram(s) - - + --time_series TIME_SERIES + Generate time series plot. Input is QATYPE-METRIC, + e.g. SKYSUB-MED_RESID frameqa ------- @@ -106,3 +164,22 @@ YAML file of all the QA outputs:: desi_qa_prod --slurp # Collate all the QA YAML files desi_qa_prod --slurp --remove # Collate and remove the individual files + +Channel Histograms +------------------ + +Using the --channel_hist flag, the script will generate a series +of histogram plots on default metrics: FIBERFLAT: MAX_RMS, +SKYSUB: MED_RESID, FLUXCALIB: MAX_ZP_OFF:: + + desi_qa_prod --channel_hist + +Time Series Plot +---------------- + +Using the --time_series input with a *qatype* and *metric* produces +a Time Series plot of that metric for all nights/exposures/frames +in the production, by channel, e.g.:: + + desi_qa_prod --time_series=SKYSUB-MED_RESID + desi_qa_prod --time_series=FLUXCALIB-ZP diff --git a/py/desispec/io/meta.py b/py/desispec/io/meta.py index 2a14fd2b6..4d0914582 100644 --- a/py/desispec/io/meta.py +++ b/py/desispec/io/meta.py @@ -300,26 +300,27 @@ def get_exposures(night, raw=False, rawdata_dir=None, specprod_dir=None): return sorted(exposures) -def get_reduced_frames(channels=['b','r','z'], nights=None): +def get_reduced_frames(channels=['b','r','z'], nights=None, ftype='cframe'): """ Loops through a production to find all reduced frames (default is cframes) - One can choose a subset by argument + One can choose a subset of reduced frames by argument Args: channels: list, optional nights: list, optional + ftype: str, optional Returns: all_frames: list for frame filenames """ all_frames = [] + # Nights if nights is None: - paths_to_nights = glob.glob(specprod_root()+'/exposures/*') - nights = [ipathn[ipathn.rfind('/')+1:] for ipathn in paths_to_nights] + nights = get_nights() # Loop on night for night in nights: exposures = get_exposures(night) for exposure in exposures: - frames_dict = get_files(filetype=str('cframe'), night=night, expid=exposure) + frames_dict = get_files(filetype=ftype, night=night, expid=exposure) # Restrict on channel for key in frames_dict.keys(): for channel in channels: diff --git a/py/desispec/scripts/skysubresid.py b/py/desispec/scripts/skysubresid.py index 4a977cf70..bedf2146e 100644 --- a/py/desispec/scripts/skysubresid.py +++ b/py/desispec/scripts/skysubresid.py @@ -7,15 +7,15 @@ def parse(options=None): - parser = argparse.ArgumentParser(description="Generate QA on Sky Subtraction residuals") + parser = argparse.ArgumentParser(description="Generate QA on Sky Subtraction residuals [v1.2]") parser.add_argument('--reduxdir', type = str, default = None, metavar = 'PATH', help = 'Override default path ($DESI_SPECTRO_REDUX/$SPECPROD) to processed data.') parser.add_argument('--expid', type=int, help='Generate exposure plot on given exposure') parser.add_argument('--night', type=str, help='Generate night plot on given night') - parser.add_argument('--nights', type=str, help='List of nights to include for prod plot') parser.add_argument('--channels', type=str, help='List of channels to include') parser.add_argument('--prod', default=False, action="store_true", help="Results for full production run") parser.add_argument('--gauss', default=False, action="store_true", help="Expore Gaussianity for full production run") + parser.add_argument('--nights', type=str, help='List of nights to limit prod plots') if options is None: args = parser.parse_args() @@ -30,7 +30,7 @@ def main(args) : import glob from desispec.io import findfile from desispec.io import get_exposures - from desispec.io import get_files + from desispec.io import get_files, get_nights from desispec.io import read_frame from desispec.io import get_reduced_frames from desispec.io.sky import read_sky @@ -49,11 +49,6 @@ def main(args) : else: specprod_dir = specprod_root() - # Nights - if args.nights is not None: - nights = [iarg for iarg in args.nights.split(',')] - else: - nights = None # Channels if args.channels is not None: @@ -73,7 +68,7 @@ def main(args) : # Nights path_nights = glob.glob(specprod_dir+'/exposures/*') if nights is None: - nights = [ipathn[ipathn.rfind('/')+1:] for ipathn in path_nights] + nights = get_nights() nights.sort() # Find the exposure for night in nights: @@ -104,6 +99,11 @@ def main(args) : return + # Nights + if args.nights is not None: + nights = [iarg for iarg in args.nights.split(',')] + else: + nights = None # Full Prod Plot? if args.prod: from desispec.qa.qa_plots import skysub_resid_dual @@ -117,6 +117,7 @@ def main(args) : log.info("Plotting..") skysub_resid_dual(sky_wave, sky_flux, sky_res, outfile='skyresid_prod_dual_{:s}.png'.format(channel)) + return # Full Prod Plot? if args.gauss: @@ -134,3 +135,4 @@ def main(args) : log.info("Plotting..") skysub_gauss(sky_wave, sky_flux, sky_res, sky_ivar, outfile='skyresid_prod_gauss_{:s}.png'.format(channel)) + return diff --git a/py/desispec/test/test_io.py b/py/desispec/test/test_io.py index 589d482d8..37e127877 100644 --- a/py/desispec/test/test_io.py +++ b/py/desispec/test/test_io.py @@ -622,6 +622,25 @@ def test_search_framefile(self): mfile = search_for_framefile('frame-b0-000123.fits') self.assertEqual(x, mfile) + def test_get_reduced_frames(self): + """ Test desispec.io.get_reduced_frames + """ + from ..io import get_reduced_frames + from ..io.meta import findfile + from ..io.util import makepath + # Setup paths + os.environ['DESI_SPECTRO_REDUX'] = self.testEnv['DESI_SPECTRO_REDUX'] + os.environ['SPECPROD'] = self.testEnv['SPECPROD'] + # Generate a dummy frame file + for expid, night in zip((123,150), ['20150101', '20150102']): + x = findfile('cframe', camera='b0', night=night, expid=expid) + makepath(x) + with open(x,'a') as f: + pass + # Find it + mfile = get_reduced_frames() + self.assertEqual(2, len(mfile)) + @unittest.skipUnless(os.path.exists(os.path.join(os.environ['HOME'],'.netrc')),"No ~/.netrc file detected.") def test_download(self): """Test desiutil.io.download.