From edde300ba34c597a5b01aa69a94bb897e81821f5 Mon Sep 17 00:00:00 2001 From: anand_raichoor Date: Thu, 16 Dec 2021 00:41:38 -0800 Subject: [PATCH 01/13] allow other surveys than main --- bin/desi_night_qa | 13 +- py/desispec/night_qa.py | 289 ++++++++++++++++++++++++---------------- 2 files changed, 179 insertions(+), 123 deletions(-) diff --git a/bin/desi_night_qa b/bin/desi_night_qa index 095b7241a..ba12754b4 100755 --- a/bin/desi_night_qa +++ b/bin/desi_night_qa @@ -10,13 +10,14 @@ This script generates the $DESI_ROOT/spectro/redux/nightqa/{NIGHT}/nightqa-{NIGH import os,sys +import numpy as np import argparse from desiutil.log import get_logger from desispec.io import specprod_root from pkg_resources import resource_filename from desispec.night_qa import ( get_nightqa_outfns, - get_survey_night_expids, + get_surveys_night_expids, get_dark_night_expid, get_ctedet_night_expid, create_dark_pdf, @@ -94,7 +95,7 @@ def main(): sys.exit(1) # AR expids, tileids - expids, tileids = get_survey_night_expids(args.night, "main") + expids, tileids, surveys = get_surveys_night_expids(args.night) dark_expid = get_dark_night_expid(args.night) ctedet_expid = get_ctedet_night_expid(args.night, args.prod) @@ -116,17 +117,19 @@ def main(): create_tileqa_pdf(outfns["tileqa"], args.night, args.prod, expids, tileids) # AR skyzfiber - create_skyzfiber_png(outfns["skyzfiber"], args.night, args.prod, survey="main", dchi2_threshold=9) + create_skyzfiber_png(outfns["skyzfiber"], args.night, args.prod, np.unique(tileids), dchi2_threshold=9) # AR per-petal n(z) - create_petalnz_pdf(outfns["petalnz"], args.night, args.prod, survey="main", dchi2_threshold=25) + unq_tileids, ii = np.unique(tileids, return_index=True) + unq_surveys = surveys[ii] + create_petalnz_pdf(outfns["petalnz"], args.night, args.prod, unq_tileids, unq_surveys, dchi2_threshold=25) # AR create index.html # AR we first copy the args.css file to args.outdir os.system("cp {} {}".format(args.css, args.outdir)) write_nightqa_html( outfns, args.night, args.prod, os.path.basename(args.css), - survey="main", nexp=expids.size, ntile=len(set(tileids))) + surveys="/".join(np.unique(surveys)), nexp=expids.size, ntile=len(set(tileids))) if __name__ == "__main__": main() diff --git a/py/desispec/night_qa.py b/py/desispec/night_qa.py index 276ace9ea..8138c40bd 100644 --- a/py/desispec/night_qa.py +++ b/py/desispec/night_qa.py @@ -16,6 +16,7 @@ # AR desitarget from desitarget.targetmask import desi_mask, bgs_mask from desitarget.targetmask import zwarn_mask as desitarget_zwarn_mask +from desitarget.targets import main_cmx_or_sv from desitarget.targets import zcut as lya_zcut # AR desispec from desispec.fiberbitmasking import get_skysub_fiberbitmask_val @@ -54,21 +55,21 @@ def get_nightqa_outfns(outdir, night): -def get_survey_night_expids( +def get_surveys_night_expids( night, - survey, datadir = None): """ List the (EXPIDs, TILEIDs) from a given night for a given survey. Args: night: night (int) - survey: "main", "sv3", "sv2", or "sv1" (str) + surveys: comma-separated list of surveys to consider, in lower-cases, e.g. "sv1,sv2,sv3,main" (str) datadir (optional, defaults to $DESI_SPECTRO_DATA): full path where the {NIGHT}/desi-{EXPID}.fits.fz files are (str) Returns: expids: list of the EXPIDs (np.array()) tileids: list of the TILEIDs (np.array()) + surveys: list of the SURVEYs (np.array()) Notes: Based on parsing the OBSTYPE and NTSSURVY keywords from the SPEC extension header of the desi-{EXPID}.fits.fz files. @@ -85,19 +86,46 @@ def get_survey_night_expids( ) ) ) - expids, tileids = [], [] + expids, tileids, surveys = [], [], [] for i in range(len(fns)): hdr = fits.getheader(fns[i], "SPEC") if hdr["OBSTYPE"] == "SCIENCE": - if hdr["NTSSURVY"] == survey: - expids.append(hdr["EXPID"]) - tileids.append(hdr["TILEID"]) - log.info( - "found {} exposures from {} tiles for SURVEY={} and NIGHT={}".format( - len(expids), np.unique(tileids).size, survey, night, + survey = "unknown" + # AR first try the NTSSURVY keyword + # AR - discard cases where NTSSURVY="" or "na"... + if "NTSSURVY" in [cards[0] for cards in hdr.cards]: + if hdr["NTSSURVY"] not in ["", "na"]: + survey = hdr["NTSSURVY"] + # AR else look for the fiberassign file + # AR - used wildcard, because early files (pre-SV1?) were not gzipped + # AR - first check SURVEY keyword (should work for SV3 and later) + # AR - if not present, take FA_SURV + if survey == "unknown": + fafns = glob(os.path.join(os.path.dirname(fns[i]), "fiberassign-??????.fits*")) + if len(fafns) > 0: + fahdr = fits.getheader(fafns[0], 0) + if "SURVEY" in [cards[0] for cards in fahdr.cards]: + survey = fahdr["SURVEY"] + else: + survey = fahdr["FA_SURV"] + if survey is None: + log.warning("SURVEY could not be identified for {}; setting to 'unknown'".format(fns[i])) + # AR append + expids.append(hdr["EXPID"]) + tileids.append(hdr["TILEID"]) + surveys.append(survey) + expids, tileids, surveys = np.array(expids), np.array(tileids), np.array(surveys) + msg = "for NIGHT={} found" + per_surv = [] + for survey in np.unique(surveys): + sel = surveys == survey + per_surv.append( + "{} exposures from {} tiles for SURVEY={}".format( + sel.sum(), np.unique(tileids[sel]).size, survey, + ) ) - ) - return np.array(expids), np.array(tileids) + log.info("for NIGHT={} found {}".format(night, " and ".join(per_surv))) + return expids, tileids, surveys def get_dark_night_expid(night, datadir = None): @@ -657,7 +685,7 @@ def create_tileqa_pdf(outpdf, night, prod, expids, tileids): plt.close() -def create_skyzfiber_png(outpng, night, prod, survey="main", dchi2_threshold=9): +def create_skyzfiber_png(outpng, night, prod, tileids, dchi2_threshold=9): """ For a given night, create a Z vs. FIBER plot for all SKY fibers. @@ -665,14 +693,13 @@ def create_skyzfiber_png(outpng, night, prod, survey="main", dchi2_threshold=9): outpdf: output pdf file (string) night: night (int) prod: full path to prod folder, e.g. /global/cfs/cdirs/desi/spectro/redux/blanc/ (string) - survey (optional, defaults to "main"): survey from which pick tileids + tileids: list of tileids to consider (list or numpy array) dchi2_threshold (optional, defaults to 9): DELTACHI2 value to split the sample (float) Notes: Work from the redrock*fits files. """ - # AR pick the tileids - _, tileids = get_survey_night_expids(night, survey) + # AR safe tileids = np.unique(tileids) # AR gather all infos from the redrock*fits files fibers, zs, dchi2s = [], [], [] @@ -725,7 +752,7 @@ def create_skyzfiber_png(outpng, night, prod, survey="main", dchi2_threshold=9): plt.close() -def create_petalnz_pdf(outpdf, night, prod, survey="main", dchi2_threshold=25): +def create_petalnz_pdf(outpdf, night, prod, tileids, surveys, dchi2_threshold=25): """ For a given night, create a per-petal, per-tracer n(z) pdf file. @@ -733,33 +760,38 @@ def create_petalnz_pdf(outpdf, night, prod, survey="main", dchi2_threshold=25): outpdf: output pdf file (string) night: night (int) prod: full path to prod folder, e.g. /global/cfs/cdirs/desi/spectro/redux/blanc/ (string) - survey (optional, defaults to "main"): survey from which pick tileids + tileids: list of tileids to consider (list or numpy array) + surveys: list of the surveys for each tileid of tileids (list or numpy array) dchi2_threshold (optional, defaults to 9): DELTACHI2 value to split the sample (float) Notes: - Work from the zmtl*fits files, trying to mimick what is done in desitarget.mtl.make_mtl(). + Only displays: + - sv1, sv2, sv3, main, as otherwise the plotted tracers are not relevant; + - FAPRGRM="bright" or "dark" tileids. + If the tile-qa-TILEID-thruNIGHT.fits file is missing, that tileid is skipped. + For the Lya, work from the zmtl*fits files, trying to mimick what is done in desitarget.mtl.make_mtl(). The LRG, ELG, QSO, BGS_BRIGHT, BGS_FAINT bit are the same for sv1, sv2, sv3, main, so ok to simply use the bit mask values from the main. TBD : we query the FAPRGRM of the tile-qa-*fits header, not sure that properly works for surveys other than main.. """ petals = np.arange(10, dtype=int) - if survey not in ["sv1", "sv2", "sv3", "main"]: - log.warning("survey = {} not in sv1, sv2, sv3, main, not plotting".format(survey)) - return - # AR column name - if survey == "main": - prefix_key = "" - else: - prefix_key = "{}_".format(survey.upper()) - # AR pick the tileids - _, tileids = get_survey_night_expids(night, survey) - tileids = np.unique(tileids) + # AR safe + tileids, ii = np.unique(tileids, return_index=True) + surveys = surveys[ii] + # AR cutting on sv1, sv2, sv3, main + sel = np.in1d(surveys, ["sv1", "sv2", "sv3", "main"]) + log.info( + "removing {}/{} tileids corresponding to surveys={}, different than sv1, sv2, sv3, main".format( + (~sel).sum(), tileids.size, ",".join(np.unique(surveys[~sel]).astype(str)), + ) + ) + tileids, surveys = tileids[sel], surveys[sel] # # AR gather all infos from the zmtl*fits files ds = {"bright" : [], "dark" : []} ntiles = {"bright" : 0, "dark" : 0} - for tileid in tileids: + for tileid, survey in zip(tileids, surveys): # AR bright or dark? fn = os.path.join( prod, @@ -769,7 +801,7 @@ def create_petalnz_pdf(outpdf, night, prod, survey="main", dchi2_threshold=25): "{}".format(night), "tile-qa-{}-thru{}.fits".format(tileid, night), ) - # AR trying to protect against non-main tiles... + # AR if no tile-qa*fits, we skip the tileid if not os.path.isfile(fn): log.warning("no {} file, proceeding to next tile".format(fn)) continue @@ -777,9 +809,9 @@ def create_petalnz_pdf(outpdf, night, prod, survey="main", dchi2_threshold=25): if "FAPRGRM" not in [cards[0] for cards in hdr.cards]: log.warning("no FAPRGRM in {} header, proceeding to next tile".format(fn)) continue - faprgrm = hdr["FAPRGRM"] + faprgrm = hdr["FAPRGRM"].lower() if faprgrm not in ["bright", "dark"]: - log.warning("FAPRGRM={} not in bright, dark, proceeding to next tile".format(fn)) + log.warning("{} : FAPRGRM={} not in bright, dark, proceeding to next tile".format(fn, faprgrm)) continue # AR reading zmtl files istileid = False @@ -796,26 +828,27 @@ def create_petalnz_pdf(outpdf, night, prod, survey="main", dchi2_threshold=25): log.warning("{} : no file".format(fn)) else: istileid = True - d = Table( - fitsio.read( - fn, - ext="ZMTL", - columns=[ - "TARGETID", - "{}DESI_TARGET".format(prefix_key), "{}BGS_TARGET".format(prefix_key), - "Z", "ZWARN", "SPECTYPE", "DELTACHI2", - "Z_QN", "Z_QN_CONF", "IS_QSO_QN", - ], - ) - ) + d = Table.read(fn, hdu="ZMTL") + # AR rename *DESI_TARGET and *BGS_TARGET to DESI_TARGET and BGS_TARGET + keys, _, _ = main_cmx_or_sv(d) + d.rename_column(keys[0], "DESI_TARGET") + d.rename_column(keys[1], "BGS_TARGET") + # AR cutting on columns + d = d[ + "TARGETID", "DESI_TARGET", "BGS_TARGET", + "Z", "ZWARN", "SPECTYPE", "DELTACHI2", + "Z_QN", "Z_QN_CONF", "IS_QSO_QN", + ] + d["SURVEY"] = np.array([survey for x in range(len(d))], dtype=object) + d["TILEID"] = np.array([tileid for x in range(len(d))], dtype=int) d["PETAL_LOC"] = petal + np.zeros(len(d), dtype=int) sel = np.zeros(len(d), dtype=bool) if faprgrm == "bright": for msk in ["BGS_BRIGHT", "BGS_FAINT"]: - sel |= (d["{}BGS_TARGET".format(prefix_key)] & bgs_mask[msk]) > 0 + sel |= (d["BGS_TARGET"] & bgs_mask[msk]) > 0 if faprgrm == "dark": for msk in ["LRG", "ELG", "QSO"]: - sel |= (d["{}DESI_TARGET".format(prefix_key)] & desi_mask[msk]) > 0 + sel |= (d["DESI_TARGET"] & desi_mask[msk]) > 0 log.info("selecting {} tracer targets from {}".format(sel.sum(), fn)) d = d[sel] ds[faprgrm].append(d) @@ -850,10 +883,10 @@ def create_petalnz_pdf(outpdf, night, prod, survey="main", dchi2_threshold=25): # AR small internal plot utility function def get_tracer_props(tracer): if tracer in ["BGS_BRIGHT", "BGS_FAINT"]: - faprgrm, mask, dtkey = "bright", bgs_mask, "{}BGS_TARGET".format(prefix_key) + faprgrm, mask, dtkey = "bright", bgs_mask, "BGS_TARGET" xlim, ylim = (-0.2, 1.5), (0, 5.0) else: - faprgrm, mask, dtkey = "dark", desi_mask, "{}DESI_TARGET".format(prefix_key) + faprgrm, mask, dtkey = "dark", desi_mask, "DESI_TARGET" if tracer == "LRG": xlim, ylim = (-0.2, 2), (0, 3.0) elif tracer == "ELG": @@ -863,10 +896,6 @@ def get_tracer_props(tracer): return faprgrm, mask, dtkey, xlim, ylim # AR plot # - # AR plotting only if some tiles - doplot = False - if ntiles["bright"] + ntiles["dark"] > 0: - doplot = True # AR color for each tracer colors = { "BGS_BRIGHT" : "purple", @@ -876,22 +905,32 @@ def get_tracer_props(tracer): "QSO" : "orange", } with PdfPages(outpdf) as pdf: - # AR three plots: - # AR - fraction of VALID fibers, bright+dark together - # AR - fraction of ZOK fibers, per tracer - # AR - fraction of LYA candidates for QSOs - fig = plt.figure(figsize=(40, 5)) - gs = gridspec.GridSpec(1, 3, wspace=0.5) - title = "{} BRIGHT and {} DARK tiles from {}".format(ntiles["bright"], ntiles["dark"], night) - if doplot: + for survey in np.unique(surveys): + ntiles_surv = { + "bright" : np.unique(ds["bright"]["TILEID"][ds["bright"]["SURVEY"] == survey]).size, + "dark" : np.unique(ds["dark"]["TILEID"][ds["dark"]["SURVEY"] == survey]).size, + } + # AR plotting only if some tiles + if ntiles_surv["bright"] + ntiles_surv["dark"] == 0: + continue + # AR three plots: + # AR - fraction of VALID fibers, bright+dark together + # AR - fraction of ZOK fibers, per tracer + # AR - fraction of LYA candidates for QSOs + fig = plt.figure(figsize=(40, 5)) + gs = gridspec.GridSpec(1, 3, wspace=0.5) + title = "SURVEY={} : {} BRIGHT and {} DARK tiles from {}".format( + survey, ntiles_surv["bright"], ntiles_surv["dark"], night + ) # AR fraction of ~VALID fibers, bright+dark together ax = plt.subplot(gs[0]) ys = np.nan + np.zeros(len(petals)) for petal in petals: npet, nvalid = 0, 0 for faprgrm in faprgrms: - npet += (ds[faprgrm]["PETAL_LOC"] == petal).sum() - nvalid += ((ds[faprgrm]["PETAL_LOC"] == petal) & (ds[faprgrm]["VALID"])).sum() + issurvpet = (ds[faprgrm]["SURVEY"] == survey) & (ds[faprgrm]["PETAL_LOC"] == petal) + npet += issurvpet.sum() + nvalid += ((issurvpet) & (ds[faprgrm]["VALID"])).sum() ys[petal] = nvalid / npet ax.plot(petals, ys, "-o", color="k") ax.set_title(title) @@ -904,7 +943,9 @@ def get_tracer_props(tracer): ax = plt.subplot(gs[1]) for tracer in tracers: faprgrm, mask, dtkey, _, _ = get_tracer_props(tracer) - istracer = ((ds[faprgrm][dtkey] & mask[tracer]) > 0) & (ds[faprgrm]["VALID"]) + istracer = ds[faprgrm]["SURVEY"] == survey + istracer &= (ds[faprgrm][dtkey] & mask[tracer]) > 0 + istracer &= ds[faprgrm]["VALID"] ys = np.nan + np.zeros(len(petals)) for petal in petals: ispetal = (istracer) & (ds[faprgrm]["PETAL_LOC"] == petal) @@ -915,7 +956,10 @@ def get_tracer_props(tracer): ax.set_xlabel("PETAL_LOC") ax.set_ylabel("fraction of DELTACHI2 >_{}\n(VALID fibers only)".format(dchi2_threshold)) ax.xaxis.set_major_locator(MultipleLocator(1)) - ax.set_ylim(0.7, 1.0) + if survey == "main": + ax.set_ylim(0.7, 1.0) + else: + ax.set_ylim(0.0, 1.0) ax.grid() ax.legend() # AR - fraction of LYA candidates for QSOs @@ -924,8 +968,8 @@ def get_tracer_props(tracer): faprgrm = "dark" ys = np.nan + np.zeros(len(petals)) for petal in petals: - ispetal = (ds[faprgrm]["PETAL_LOC"] == petal) & (ds[faprgrm]["VALID"]) - isqso = (ispetal) & ((ds[faprgrm][dtkey] & desi_mask["QSO"]) > 0) + ispetsurv = (ds[faprgrm]["SURVEY"] == survey) & (ds[faprgrm]["PETAL_LOC"] == petal) & (ds[faprgrm]["VALID"]) + isqso = (ispetsurv) & ((ds[faprgrm][dtkey] & desi_mask["QSO"]) > 0) islya = (isqso) & (ds[faprgrm]["LYA"]) ys[petal] = islya.sum() / isqso.sum() ax.plot(petals, ys, "-o", color=colors["QSO"]) @@ -935,58 +979,67 @@ def get_tracer_props(tracer): ax.xaxis.set_major_locator(MultipleLocator(1)) ax.set_ylim(0, 1) ax.grid() - # - pdf.savefig(fig, bbox_inches="tight") - plt.close() - # AR per-petal, per-tracer n(z) - if doplot: + # + pdf.savefig(fig, bbox_inches="tight") + plt.close() + # AR per-petal, per-tracer n(z) for tracer in tracers: faprgrm, mask, dtkey, xlim, ylim = get_tracer_props(tracer) - istracer = ((ds[faprgrm][dtkey] & mask[tracer]) > 0) & (ds[faprgrm]["VALID"]) + istracer = ds[faprgrm]["SURVEY"] == survey + istracer &= (ds[faprgrm][dtkey] & mask[tracer]) > 0 + istracer &= ds[faprgrm]["VALID"] istracer_zok = (istracer) & (ds[faprgrm]["ZOK"]) bins = np.arange(xlim[0], xlim[1] + 0.05, 0.05) # - fig = plt.figure(figsize=(40, 5)) - gs = gridspec.GridSpec(1, 10, wspace=0.3) - for petal in petals: - ax = plt.subplot(gs[petal]) - _ = ax.hist( - ds[faprgrm]["Z"][istracer_zok], - bins=bins, - density=True, - histtype="stepfilled", - alpha=0.5, - color=colors[tracer], - label="{} All petals".format(tracer), - ) - _ = ax.hist( - ds[faprgrm]["Z"][(istracer_zok) & (ds[faprgrm]["PETAL_LOC"] == petal)], - bins=bins, - density=True, - histtype="step", - alpha=1.0, - color="k", - label="{} PETAL_LOC = {}".format(tracer, petal), - ) - ax.set_title("{} {} tiles from {}".format(ntiles[faprgrm], faprgrm.upper(), night)) - ax.set_xlabel("Z") - if petal == 0: - ax.set_ylabel("Normalized counts") - else: - ax.set_yticklabels([]) - ax.set_xlim(xlim) - ax.set_ylim(ylim) - ax.grid() - ax.set_axisbelow(True) - ax.legend(loc=1) - ax.text( - 0.97, 0.8, - "DELTACHI2 > {}".format(dchi2_threshold), - fontsize=10, fontweight="bold", color="k", - ha="right", transform=ax.transAxes, - ) - pdf.savefig(fig, bbox_inches="tight") - plt.close() + if ntiles_surv[faprgrm] > 0: + fig = plt.figure(figsize=(40, 5)) + gs = gridspec.GridSpec(1, 10, wspace=0.3) + for petal in petals: + ax = plt.subplot(gs[petal]) + _ = ax.hist( + ds[faprgrm]["Z"][istracer_zok], + bins=bins, + density=True, + histtype="stepfilled", + alpha=0.5, + color=colors[tracer], + label="{} All petals".format(tracer), + ) + _ = ax.hist( + ds[faprgrm]["Z"][(istracer_zok) & (ds[faprgrm]["PETAL_LOC"] == petal)], + bins=bins, + density=True, + histtype="step", + alpha=1.0, + color="k", + label="{} PETAL_LOC = {}".format(tracer, petal), + ) + ax.set_title( + "{} {}-{} tiles from {}".format( + ntiles_surv[faprgrm], + survey.upper(), + faprgrm.upper(), + night, + ) + ) + ax.set_xlabel("Z") + if petal == 0: + ax.set_ylabel("Normalized counts") + else: + ax.set_yticklabels([]) + ax.set_xlim(xlim) + ax.set_ylim(ylim) + ax.grid() + ax.set_axisbelow(True) + ax.legend(loc=1) + ax.text( + 0.97, 0.8, + "DELTACHI2 > {}".format(dchi2_threshold), + fontsize=10, fontweight="bold", color="k", + ha="right", transform=ax.transAxes, + ) + pdf.savefig(fig, bbox_inches="tight") + plt.close() def path_full2web(fn): @@ -1085,7 +1138,7 @@ def write_html_collapse_script(html, classname): -def write_nightqa_html(outfns, night, prod, css, survey=None, nexp=None, ntile=None): +def write_nightqa_html(outfns, night, prod, css, surveys=None, nexp=None, ntile=None): """ Write the nightqa-{NIGHT}.html page. @@ -1094,7 +1147,7 @@ def write_nightqa_html(outfns, night, prod, css, survey=None, nexp=None, ntile=N night: night (int) prod: full path to prod folder, e.g. /global/cfs/cdirs/desi/spectro/redux/blanc/ (string) css: path to the nightqa.css file - survey (optional, defaults to None): considered survey (string) + surveys (optional, defaults to None): considered surveys (string) nexp (optional, defaults to None): number of considered exposures (int) ntile (optional, defaults to None): number of considered tiles (int) """ @@ -1117,7 +1170,7 @@ def write_nightqa_html(outfns, night, prod, css, survey=None, nexp=None, ntile=N html.write("\n") html.write("\n") # - html.write("\t

For {}, {} exposures from {} {} tiles are analyzed.

\n".format(night, nexp, ntile, survey)) + html.write("\t

For {}, {} exposures from {} {} tiles are analyzed.

\n".format(night, nexp, ntile, surveys)) html.write("\t

Please click on each tab from top to bottom, and follow instructions.

\n") # AR night log From 98d3910136327f824bb4ea8afc640c4095b53eb7 Mon Sep 17 00:00:00 2001 From: anand_raichoor Date: Thu, 16 Dec 2021 01:00:46 -0800 Subject: [PATCH 02/13] get_surveys_night_expids(): not using NTSSURVY anymore --- py/desispec/night_qa.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/py/desispec/night_qa.py b/py/desispec/night_qa.py index 8138c40bd..3a5db050a 100644 --- a/py/desispec/night_qa.py +++ b/py/desispec/night_qa.py @@ -72,7 +72,9 @@ def get_surveys_night_expids( surveys: list of the SURVEYs (np.array()) Notes: - Based on parsing the OBSTYPE and NTSSURVY keywords from the SPEC extension header of the desi-{EXPID}.fits.fz files. + Based on: + - parsing the OBSTYPE keywords from the SPEC extension header of the desi-{EXPID}.fits.fz files; + - for OBSTYPE="SCIENCE", parsing the fiberassign-TILEID.fits* header """ if datadir is None: datadir = os.getenv("DESI_SPECTRO_DATA") @@ -91,24 +93,18 @@ def get_surveys_night_expids( hdr = fits.getheader(fns[i], "SPEC") if hdr["OBSTYPE"] == "SCIENCE": survey = "unknown" - # AR first try the NTSSURVY keyword - # AR - discard cases where NTSSURVY="" or "na"... - if "NTSSURVY" in [cards[0] for cards in hdr.cards]: - if hdr["NTSSURVY"] not in ["", "na"]: - survey = hdr["NTSSURVY"] - # AR else look for the fiberassign file + # AR look for the fiberassign file # AR - used wildcard, because early files (pre-SV1?) were not gzipped # AR - first check SURVEY keyword (should work for SV3 and later) # AR - if not present, take FA_SURV + fafns = glob(os.path.join(os.path.dirname(fns[i]), "fiberassign-??????.fits*")) + if len(fafns) > 0: + fahdr = fits.getheader(fafns[0], 0) + if "SURVEY" in [cards[0] for cards in fahdr.cards]: + survey = fahdr["SURVEY"] + else: + survey = fahdr["FA_SURV"] if survey == "unknown": - fafns = glob(os.path.join(os.path.dirname(fns[i]), "fiberassign-??????.fits*")) - if len(fafns) > 0: - fahdr = fits.getheader(fafns[0], 0) - if "SURVEY" in [cards[0] for cards in fahdr.cards]: - survey = fahdr["SURVEY"] - else: - survey = fahdr["FA_SURV"] - if survey is None: log.warning("SURVEY could not be identified for {}; setting to 'unknown'".format(fns[i])) # AR append expids.append(hdr["EXPID"]) From f86b756f816b75948200109702903c5bc36cd767 Mon Sep 17 00:00:00 2001 From: anand_raichoor Date: Thu, 16 Dec 2021 01:19:30 -0800 Subject: [PATCH 03/13] write_nightqa_html(): look for possible nightlog filenames --- py/desispec/night_qa.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/py/desispec/night_qa.py b/py/desispec/night_qa.py index 3a5db050a..752c2fbb7 100644 --- a/py/desispec/night_qa.py +++ b/py/desispec/night_qa.py @@ -1170,9 +1170,18 @@ def write_nightqa_html(outfns, night, prod, css, surveys=None, nexp=None, ntile= html.write("\t

Please click on each tab from top to bottom, and follow instructions.

\n") # AR night log - nighthtml = "https://data.desi.lbl.gov/desi/survey/ops/nightlogs/{}/NightSummary{}.html".format( - night, night, - ) + # AR testing different possible names + nightdir = os.path.join(os.getenv("DESI_ROOT"), "survey", "ops", "nightlogs", "{}".format(night)) + nightfn = None + for basename in [ + "NightSummary{}.html".format(night), + "nightlog_kpno.html", + "nightlog_nersc.html", + "nightlog.html", + ]: + if nightfn is None: + if os.path.isfile(os.path.join(nightdir, basename)): + nightfn = os.path.join(os.path.join(nightdir, basename)) html.write( "\n".format( night, @@ -1180,12 +1189,15 @@ def write_nightqa_html(outfns, night, prod, css, surveys=None, nexp=None, ntile= ) html.write("
\n") html.write("\t
\n") - html.write("\t

Read the nightlog for {}: {}, displayed below.

\n".format(night, nighthtml)) - html.write("\t

And consider subscribing to the desi-nightlog mailing list!\n") - html.write("\t
\n") - html.write("\t
\n") - html.write("\t\n".format(nighthtml)) - html.write("\t

And consider subscribing to the desi-nightlog mailing list!\n") + if nightfn is not None: + html.write("\t

Read the nightlog for {}: {}, displayed below.

\n".format(night, path_full2web(nightfn))) + html.write("\t

And consider subscribing to the desi-nightlog mailing list!\n") + html.write("\t
\n") + html.write("\t
\n") + html.write("\t\n".format(path_full2web(nightfn))) + html.write("\t

And consider subscribing to the desi-nightlog mailing list!\n") + else: + html.write("\t

No found nightlog for in {}

\n".format(path_full2web(nightdir))) html.write("\t
\n") html.write("
\n") html.write("\n") From 690facf960141f24283e5cabb7be77744e6772af Mon Sep 17 00:00:00 2001 From: anand_raichoor Date: Thu, 16 Dec 2021 01:48:02 -0800 Subject: [PATCH 04/13] add a --steps argument to allow to execute only some steps --- bin/desi_night_qa | 70 +++++++++++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/bin/desi_night_qa b/bin/desi_night_qa index ba12754b4..5d0971bbf 100755 --- a/bin/desi_night_qa +++ b/bin/desi_night_qa @@ -43,6 +43,8 @@ def parse(options=None): help = "html formatting css file; default to pkg_resources.resource_filename('desispec', 'data/qa/nightqa.css')") parser.add_argument("--recompute", action = "store_true", help = "recompute (i.e. overwrite args.outfile if already existing") + parser.add_argument("--steps", type = str, default = "dark,badcol,ctedet,sframesky,tileqa,skyzfiber,petalnz,html", required = False, + help = "comma-separated list of steps to execute (default=dark,badcol,ctedet,sframesky,tileqa,skyzfiber,petalnz,html)") parser.add_argument("--html_only", action = "store_true", help = "only regenerate the nightqa-{NIGHT}.html page") @@ -85,51 +87,67 @@ def main(): outfns = get_nightqa_outfns(args.outdir, args.night) # AR existing output files? if not args.html_only: - for fn in [outfns[key] for key in outfns]: - log.info("will create {}".format(fn)) - if os.path.isfile(fn): - if args.recompute: - log.warning("\texisting {} will be overwritten".format(fn)) - else: - log.error("\t{} already exists, and args.recompute = False; exiting".format(fn)) - sys.exit(1) - - # AR expids, tileids - expids, tileids, surveys = get_surveys_night_expids(args.night) - dark_expid = get_dark_night_expid(args.night) - ctedet_expid = get_ctedet_night_expid(args.night, args.prod) - - if not args.html_only: - # AR dark + for key in outfns: + fn = outfns[key] + if key in args.steps.split(","): + log.info("will create {}".format(fn)) + if os.path.isfile(fn): + if args.recompute: + log.warning("\texisting {} will be overwritten".format(fn)) + else: + log.error("\t{} already exists, and args.recompute = False; exiting".format(fn)) + sys.exit(1) + else: + log.info("{} not in args.steps={}\t=> not creating {}".format(key, args.steps, fn)) + + # AR expids, tileids, surveys + if np.in1d(["sframesky", "tileqa", "skyzfiber", "petalnz", "html"], args.steps.split(",")).sum() > 0: + expids, tileids, surveys = get_surveys_night_expids(args.night) + # AR dark expid + if np.in1d(["dark", "badcol"], args.steps.split(",")).sum() > 0: + dark_expid = get_dark_night_expid(args.night) + # AR CTE detector expid + if "ctedet" in args.steps.split(","): + ctedet_expid = get_ctedet_night_expid(args.night, args.prod) + + # AR dark + if "dark" in args.steps.split(","): create_dark_pdf(outfns["dark"], args.night, args.prod, dark_expid) - # AR badcolumn + # AR badcolumn + if "badcol" in args.steps.split(","): create_badcol_png(outfns["badcol"], args.night, args.prod) - # AR CTE detector + # AR CTE detector + if "ctedet" in args.steps.split(","): if ctedet_expid is not None: create_ctedet_pdf(outfns["ctedet"], args.night, args.prod, ctedet_expid) - # AR sframesky + # AR sframesky + if "sframesky" in args.steps.split(","): create_sframesky_pdf(outfns["sframesky"], args.night, args.prod, expids) - # AR tileqa + # AR tileqa + if "tileqa" in args.steps.split(","): create_tileqa_pdf(outfns["tileqa"], args.night, args.prod, expids, tileids) - # AR skyzfiber + # AR skyzfiber + if "skyzfiber" in args.steps.split(","): create_skyzfiber_png(outfns["skyzfiber"], args.night, args.prod, np.unique(tileids), dchi2_threshold=9) - # AR per-petal n(z) + # AR per-petal n(z) + if "petalnz" in args.steps.split(","): unq_tileids, ii = np.unique(tileids, return_index=True) unq_surveys = surveys[ii] create_petalnz_pdf(outfns["petalnz"], args.night, args.prod, unq_tileids, unq_surveys, dchi2_threshold=25) # AR create index.html # AR we first copy the args.css file to args.outdir - os.system("cp {} {}".format(args.css, args.outdir)) - write_nightqa_html( - outfns, args.night, args.prod, os.path.basename(args.css), - surveys="/".join(np.unique(surveys)), nexp=expids.size, ntile=len(set(tileids))) + if "html" in args.steps.split(","): + os.system("cp {} {}".format(args.css, args.outdir)) + write_nightqa_html( + outfns, args.night, args.prod, os.path.basename(args.css), + surveys="/".join(np.unique(surveys)), nexp=expids.size, ntile=len(set(tileids))) if __name__ == "__main__": main() From 54231e09533fe3d2a1cfd558c82ad5a88c9dd2ef Mon Sep 17 00:00:00 2001 From: anand_raichoor Date: Thu, 16 Dec 2021 01:48:49 -0800 Subject: [PATCH 05/13] protects against cases where no DARK exposure is found --- bin/desi_night_qa | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/desi_night_qa b/bin/desi_night_qa index 5d0971bbf..89e9c0aa8 100755 --- a/bin/desi_night_qa +++ b/bin/desi_night_qa @@ -112,11 +112,13 @@ def main(): # AR dark if "dark" in args.steps.split(","): - create_dark_pdf(outfns["dark"], args.night, args.prod, dark_expid) + if dark_expid is not None: + create_dark_pdf(outfns["dark"], args.night, args.prod, dark_expid) # AR badcolumn if "badcol" in args.steps.split(","): - create_badcol_png(outfns["badcol"], args.night, args.prod) + if dark_expid is not None: + create_badcol_png(outfns["badcol"], args.night, args.prod) # AR CTE detector if "ctedet" in args.steps.split(","): From 1d7720d729b362cb836a984e5f8df57279e1275f Mon Sep 17 00:00:00 2001 From: anand_raichoor Date: Thu, 16 Dec 2021 01:50:43 -0800 Subject: [PATCH 06/13] write_nightqa_html(): report if no file is present --- py/desispec/night_qa.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/py/desispec/night_qa.py b/py/desispec/night_qa.py index 752c2fbb7..46ba87925 100644 --- a/py/desispec/night_qa.py +++ b/py/desispec/night_qa.py @@ -1292,22 +1292,26 @@ def write_nightqa_html(outfns, night, prod, css, surveys=None, nexp=None, ntile= ) html.write("
\n") html.write("\t
\n") - for text_split in text.split("\n"): - html.write("\t

{}

\n".format(text_split)) - html.write("\t\n") - if os.path.splitext(outfns[case])[-1] == ".png": - outpng = path_full2web(outfns[case]) - html.write( - "\t\n".format( - outpng, outpng, width, + print(outfns[case], os.path.isfile(outfns[case])) + if os.path.isfile(outfns[case]): + for text_split in text.split("\n"): + html.write("\t

{}

\n".format(text_split)) + html.write("\t\n") + if os.path.splitext(outfns[case])[-1] == ".png": + outpng = path_full2web(outfns[case]) + html.write( + "\t\n".format( + outpng, outpng, width, + ) ) - ) - elif os.path.splitext(outfns[case])[-1] == ".pdf": - outpdf = path_full2web(outfns[case]) - html.write("\t\n".format(outpdf, width)) + elif os.path.splitext(outfns[case])[-1] == ".pdf": + outpdf = path_full2web(outfns[case]) + html.write("\t\n".format(outpdf, width)) + else: + log.error("Unexpected extension for {}; exiting".format(outfns[case])) + sys.exit(1) else: - log.error("Unexpected extension for {}; exiting".format(outfns[case])) - sys.exit(1) + html.write("\t

No {}.

\n".format(path_full2web(outfns[case]))) html.write("\t
\n") html.write("
\n") html.write("\n") From 92820e22b98750f132466af942f068f246209eed Mon Sep 17 00:00:00 2001 From: anand_raichoor Date: Thu, 16 Dec 2021 01:51:44 -0800 Subject: [PATCH 07/13] write_nightqa_html(): small re-writing --- py/desispec/night_qa.py | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/py/desispec/night_qa.py b/py/desispec/night_qa.py index 46ba87925..4e0d39e54 100644 --- a/py/desispec/night_qa.py +++ b/py/desispec/night_qa.py @@ -1249,23 +1249,8 @@ def write_nightqa_html(outfns, night, prod, css, surveys=None, nexp=None, ntile= html.write("\n") html.write("\n") - # AR DARK - html.write( - "\n".format( - night, - ) - ) - html.write("
\n") - html.write("\t
\n") - html.write("\t

This pdf displays the 300s (binned) DARK (one page per spectrograph; non-valid pixels are displayed in red).

\n") - html.write("\t

Watch it and report unsual features (easy to say!)

\n") - html.write("\t\n") - html.write("\t\n".format(path_full2web(outfns["dark"]))) - html.write("\t
\n") - html.write("
\n") - html.write("\n") - # AR various tabs: + # AR - dark # AR - badcol # AR - ctedet # AR - sframesky @@ -1273,10 +1258,11 @@ def write_nightqa_html(outfns, night, prod, css, surveys=None, nexp=None, ntile= # AR - skyzfiber # AR - petalnz for case, caselab, width, text in zip( - ["badcol", "ctedet", "sframesky", "tileqa", "skyzfiber", "petalnz"], - ["bad columns", "CTE detector", "sframesky", "Tile QA", "SKY Z vs. FIBER", "Per-petal n(z)"], - ["35%", "100%", "75%", "90%", "35%", "100%"], + ["dark", "badcol", "ctedet", "sframesky", "tileqa", "skyzfiber", "petalnz"], + ["DARK", "bad columns", "CTE detector", "sframesky", "Tile QA", "SKY Z vs. FIBER", "Per-petal n(z)"], + ["100%", "35%", "100%", "75%", "90%", "35%", "100%"], [ + "This pdf displays the 300s (binned) DARK (one page per spectrograph; non-valid pixels are displayed in red)\nWatch it and report unsual features (easy to say!)", "This plot displays the histograms of the bad columns.\nWatch it and report unsual features (easy to say!)", "This pdf displays a small diagnosis to detect CTE anormal behaviour (one petal-camera per page)\nWatch it and report unusual features (typically if the lower enveloppe of the blue or orange curve is systematically lower than the other one).", "This pdf displays the sframe image for the sky fibers for each Main exposure (one exposure per page).\nWatch it and report unsual features (easy to say!)", From af8913d4430c12bd19013aa98754d63283df74d5 Mon Sep 17 00:00:00 2001 From: anand_raichoor Date: Thu, 16 Dec 2021 01:57:15 -0800 Subject: [PATCH 08/13] replace sys.exit(1) by some raise Errors --- bin/desi_night_qa | 8 ++++---- py/desispec/night_qa.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bin/desi_night_qa b/bin/desi_night_qa index 89e9c0aa8..361d14c3f 100755 --- a/bin/desi_night_qa +++ b/bin/desi_night_qa @@ -76,8 +76,8 @@ def main(): # AR is ffmpeg installed # AR disabled for now, as using pdf; keep the lines in case we generate mp4 later # if os.system("which ffmpeg") != 0: - # log.error("ffmpeg needs to be installed to create the mp4 movies; it can be installed at nersc with 'module load ffmpeg'; exiting") - # sys.exit(1) + # log.error("ffmpeg needs to be installed to create the mp4 movies; it can be installed at nersc with 'module load ffmpeg'") + # raise RuntimeError("ffmpeg needs to be installed to create the mp4 movies; it can be installed at nersc with 'module load ffmpeg'") # AR existing output folder? if not os.path.isdir(args.outdir): @@ -95,8 +95,8 @@ def main(): if args.recompute: log.warning("\texisting {} will be overwritten".format(fn)) else: - log.error("\t{} already exists, and args.recompute = False; exiting".format(fn)) - sys.exit(1) + log.error("\t{} already exists, and args.recompute = False".format(fn)) + raise RuntimeError("\t{} already exists, and args.recompute = False".format(fn)) else: log.info("{} not in args.steps={}\t=> not creating {}".format(key, args.steps, fn)) diff --git a/py/desispec/night_qa.py b/py/desispec/night_qa.py index 4e0d39e54..95745646b 100644 --- a/py/desispec/night_qa.py +++ b/py/desispec/night_qa.py @@ -288,8 +288,8 @@ def create_mp4(fns, outmp4, duration=15): """ # AR is ffmpeg installed if os.system("which ffmpeg") != 0: - log.error("ffmpeg needs to be installed to run create_mp4(); exiting") - sys.exit(1) + log.error("ffmpeg needs to be installed to create the mp4 movies; it can be installed at nersc with 'module load ffmpeg'") + raise RuntimeError("ffmpeg needs to be installed to run create_mp4()") # AR deleting existing video mp4, if any if os.path.isfile(outmp4): log.info("deleting existing {}".format(outmp4)) @@ -1294,8 +1294,8 @@ def write_nightqa_html(outfns, night, prod, css, surveys=None, nexp=None, ntile= outpdf = path_full2web(outfns[case]) html.write("\t\n".format(outpdf, width)) else: - log.error("Unexpected extension for {}; exiting".format(outfns[case])) - sys.exit(1) + log.error("Unexpected extension for {}".format(outfns[case])) + raise RuntimeError("Unexpected extension for {}".format(outfns[case])) else: html.write("\t

No {}.

\n".format(path_full2web(outfns[case]))) html.write("\t
\n") From 1520448998a3569a2a4cf8dd178e6996dd842d8b Mon Sep 17 00:00:00 2001 From: anand_raichoor Date: Thu, 16 Dec 2021 02:18:41 -0800 Subject: [PATCH 09/13] remove leftover print() --- py/desispec/night_qa.py | 1 - 1 file changed, 1 deletion(-) diff --git a/py/desispec/night_qa.py b/py/desispec/night_qa.py index 95745646b..b6c7a8056 100644 --- a/py/desispec/night_qa.py +++ b/py/desispec/night_qa.py @@ -1278,7 +1278,6 @@ def write_nightqa_html(outfns, night, prod, css, surveys=None, nexp=None, ntile= ) html.write("
\n") html.write("\t
\n") - print(outfns[case], os.path.isfile(outfns[case])) if os.path.isfile(outfns[case]): for text_split in text.split("\n"): html.write("\t

{}

\n".format(text_split)) From 3722e995d19069bfff9d9fbcbdf8525ad496f0f4 Mon Sep 17 00:00:00 2001 From: anand_raichoor Date: Thu, 16 Dec 2021 02:29:36 -0800 Subject: [PATCH 10/13] create_petalnz_pdf(): minor code change --- py/desispec/night_qa.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/py/desispec/night_qa.py b/py/desispec/night_qa.py index b6c7a8056..b4ce24ea6 100644 --- a/py/desispec/night_qa.py +++ b/py/desispec/night_qa.py @@ -777,12 +777,13 @@ def create_petalnz_pdf(outpdf, night, prod, tileids, surveys, dchi2_threshold=25 surveys = surveys[ii] # AR cutting on sv1, sv2, sv3, main sel = np.in1d(surveys, ["sv1", "sv2", "sv3", "main"]) - log.info( - "removing {}/{} tileids corresponding to surveys={}, different than sv1, sv2, sv3, main".format( - (~sel).sum(), tileids.size, ",".join(np.unique(surveys[~sel]).astype(str)), + if sel.sum() > 0: + log.info( + "removing {}/{} tileids corresponding to surveys={}, different than sv1, sv2, sv3, main".format( + (~sel).sum(), tileids.size, ",".join(np.unique(surveys[~sel]).astype(str)), + ) ) - ) - tileids, surveys = tileids[sel], surveys[sel] + tileids, surveys = tileids[sel], surveys[sel] # # AR gather all infos from the zmtl*fits files ds = {"bright" : [], "dark" : []} From 0c3bad26012337027b166886f125959492970fc6 Mon Sep 17 00:00:00 2001 From: anand_raichoor Date: Thu, 16 Dec 2021 21:46:53 -0800 Subject: [PATCH 11/13] create_petalnz_pdf(): handle cases with no tiles --- py/desispec/night_qa.py | 264 ++++++++++++++++++++-------------------- 1 file changed, 133 insertions(+), 131 deletions(-) diff --git a/py/desispec/night_qa.py b/py/desispec/night_qa.py index b4ce24ea6..429d91259 100644 --- a/py/desispec/night_qa.py +++ b/py/desispec/night_qa.py @@ -902,141 +902,143 @@ def get_tracer_props(tracer): "QSO" : "orange", } with PdfPages(outpdf) as pdf: - for survey in np.unique(surveys): - ntiles_surv = { - "bright" : np.unique(ds["bright"]["TILEID"][ds["bright"]["SURVEY"] == survey]).size, - "dark" : np.unique(ds["dark"]["TILEID"][ds["dark"]["SURVEY"] == survey]).size, - } - # AR plotting only if some tiles - if ntiles_surv["bright"] + ntiles_surv["dark"] == 0: - continue - # AR three plots: - # AR - fraction of VALID fibers, bright+dark together - # AR - fraction of ZOK fibers, per tracer - # AR - fraction of LYA candidates for QSOs - fig = plt.figure(figsize=(40, 5)) - gs = gridspec.GridSpec(1, 3, wspace=0.5) - title = "SURVEY={} : {} BRIGHT and {} DARK tiles from {}".format( - survey, ntiles_surv["bright"], ntiles_surv["dark"], night - ) - # AR fraction of ~VALID fibers, bright+dark together - ax = plt.subplot(gs[0]) - ys = np.nan + np.zeros(len(petals)) - for petal in petals: - npet, nvalid = 0, 0 - for faprgrm in faprgrms: - issurvpet = (ds[faprgrm]["SURVEY"] == survey) & (ds[faprgrm]["PETAL_LOC"] == petal) - npet += issurvpet.sum() - nvalid += ((issurvpet) & (ds[faprgrm]["VALID"])).sum() - ys[petal] = nvalid / npet - ax.plot(petals, ys, "-o", color="k") - ax.set_title(title) - ax.set_xlabel("PETAL_LOC") - ax.set_ylabel("fraction of VALID_fibers") - ax.xaxis.set_major_locator(MultipleLocator(1)) - ax.set_ylim(0.5, 1.0) - ax.grid() - # AR - fraction of ZOK fibers, per tracer (VALID fibers only) - ax = plt.subplot(gs[1]) - for tracer in tracers: - faprgrm, mask, dtkey, _, _ = get_tracer_props(tracer) - istracer = ds[faprgrm]["SURVEY"] == survey - istracer &= (ds[faprgrm][dtkey] & mask[tracer]) > 0 - istracer &= ds[faprgrm]["VALID"] - ys = np.nan + np.zeros(len(petals)) - for petal in petals: - ispetal = (istracer) & (ds[faprgrm]["PETAL_LOC"] == petal) - iszok = (ispetal) & (ds[faprgrm]["ZOK"]) - ys[petal] = iszok.sum() / ispetal.sum() - ax.plot(petals, ys, "-o", color=colors[tracer], label=tracer) - ax.set_title(title) - ax.set_xlabel("PETAL_LOC") - ax.set_ylabel("fraction of DELTACHI2 >_{}\n(VALID fibers only)".format(dchi2_threshold)) - ax.xaxis.set_major_locator(MultipleLocator(1)) - if survey == "main": - ax.set_ylim(0.7, 1.0) - else: - ax.set_ylim(0.0, 1.0) - ax.grid() - ax.legend() - # AR - fraction of LYA candidates for QSOs - ax = plt.subplot(gs[2]) - if "dark" in faprgrms: - faprgrm = "dark" + # AR we need some tiles to plot! + if ntiles["bright"] + ntiles["dark"] > 0: + for survey in np.unique(surveys): + ntiles_surv = { + "bright" : np.unique(ds["bright"]["TILEID"][ds["bright"]["SURVEY"] == survey]).size, + "dark" : np.unique(ds["dark"]["TILEID"][ds["dark"]["SURVEY"] == survey]).size, + } + # AR plotting only if some tiles + if ntiles_surv["bright"] + ntiles_surv["dark"] == 0: + continue + # AR three plots: + # AR - fraction of VALID fibers, bright+dark together + # AR - fraction of ZOK fibers, per tracer + # AR - fraction of LYA candidates for QSOs + fig = plt.figure(figsize=(40, 5)) + gs = gridspec.GridSpec(1, 3, wspace=0.5) + title = "SURVEY={} : {} BRIGHT and {} DARK tiles from {}".format( + survey, ntiles_surv["bright"], ntiles_surv["dark"], night + ) + # AR fraction of ~VALID fibers, bright+dark together + ax = plt.subplot(gs[0]) ys = np.nan + np.zeros(len(petals)) for petal in petals: - ispetsurv = (ds[faprgrm]["SURVEY"] == survey) & (ds[faprgrm]["PETAL_LOC"] == petal) & (ds[faprgrm]["VALID"]) - isqso = (ispetsurv) & ((ds[faprgrm][dtkey] & desi_mask["QSO"]) > 0) - islya = (isqso) & (ds[faprgrm]["LYA"]) - ys[petal] = islya.sum() / isqso.sum() - ax.plot(petals, ys, "-o", color=colors["QSO"]) - ax.set_title(title) - ax.set_xlabel("PETAL_LOC") - ax.set_ylabel("fraction of LYA candidates\n(VALID QSO fibers only)") - ax.xaxis.set_major_locator(MultipleLocator(1)) - ax.set_ylim(0, 1) - ax.grid() - # - pdf.savefig(fig, bbox_inches="tight") - plt.close() - # AR per-petal, per-tracer n(z) - for tracer in tracers: - faprgrm, mask, dtkey, xlim, ylim = get_tracer_props(tracer) - istracer = ds[faprgrm]["SURVEY"] == survey - istracer &= (ds[faprgrm][dtkey] & mask[tracer]) > 0 - istracer &= ds[faprgrm]["VALID"] - istracer_zok = (istracer) & (ds[faprgrm]["ZOK"]) - bins = np.arange(xlim[0], xlim[1] + 0.05, 0.05) - # - if ntiles_surv[faprgrm] > 0: - fig = plt.figure(figsize=(40, 5)) - gs = gridspec.GridSpec(1, 10, wspace=0.3) + npet, nvalid = 0, 0 + for faprgrm in faprgrms: + issurvpet = (ds[faprgrm]["SURVEY"] == survey) & (ds[faprgrm]["PETAL_LOC"] == petal) + npet += issurvpet.sum() + nvalid += ((issurvpet) & (ds[faprgrm]["VALID"])).sum() + ys[petal] = nvalid / npet + ax.plot(petals, ys, "-o", color="k") + ax.set_title(title) + ax.set_xlabel("PETAL_LOC") + ax.set_ylabel("fraction of VALID_fibers") + ax.xaxis.set_major_locator(MultipleLocator(1)) + ax.set_ylim(0.5, 1.0) + ax.grid() + # AR - fraction of ZOK fibers, per tracer (VALID fibers only) + ax = plt.subplot(gs[1]) + for tracer in tracers: + faprgrm, mask, dtkey, _, _ = get_tracer_props(tracer) + istracer = ds[faprgrm]["SURVEY"] == survey + istracer &= (ds[faprgrm][dtkey] & mask[tracer]) > 0 + istracer &= ds[faprgrm]["VALID"] + ys = np.nan + np.zeros(len(petals)) for petal in petals: - ax = plt.subplot(gs[petal]) - _ = ax.hist( - ds[faprgrm]["Z"][istracer_zok], - bins=bins, - density=True, - histtype="stepfilled", - alpha=0.5, - color=colors[tracer], - label="{} All petals".format(tracer), - ) - _ = ax.hist( - ds[faprgrm]["Z"][(istracer_zok) & (ds[faprgrm]["PETAL_LOC"] == petal)], - bins=bins, - density=True, - histtype="step", - alpha=1.0, - color="k", - label="{} PETAL_LOC = {}".format(tracer, petal), - ) - ax.set_title( - "{} {}-{} tiles from {}".format( - ntiles_surv[faprgrm], - survey.upper(), - faprgrm.upper(), - night, + ispetal = (istracer) & (ds[faprgrm]["PETAL_LOC"] == petal) + iszok = (ispetal) & (ds[faprgrm]["ZOK"]) + ys[petal] = iszok.sum() / ispetal.sum() + ax.plot(petals, ys, "-o", color=colors[tracer], label=tracer) + ax.set_title(title) + ax.set_xlabel("PETAL_LOC") + ax.set_ylabel("fraction of DELTACHI2 >_{}\n(VALID fibers only)".format(dchi2_threshold)) + ax.xaxis.set_major_locator(MultipleLocator(1)) + if survey == "main": + ax.set_ylim(0.7, 1.0) + else: + ax.set_ylim(0.0, 1.0) + ax.grid() + ax.legend() + # AR - fraction of LYA candidates for QSOs + ax = plt.subplot(gs[2]) + if "dark" in faprgrms: + faprgrm = "dark" + ys = np.nan + np.zeros(len(petals)) + for petal in petals: + ispetsurv = (ds[faprgrm]["SURVEY"] == survey) & (ds[faprgrm]["PETAL_LOC"] == petal) & (ds[faprgrm]["VALID"]) + isqso = (ispetsurv) & ((ds[faprgrm][dtkey] & desi_mask["QSO"]) > 0) + islya = (isqso) & (ds[faprgrm]["LYA"]) + ys[petal] = islya.sum() / isqso.sum() + ax.plot(petals, ys, "-o", color=colors["QSO"]) + ax.set_title(title) + ax.set_xlabel("PETAL_LOC") + ax.set_ylabel("fraction of LYA candidates\n(VALID QSO fibers only)") + ax.xaxis.set_major_locator(MultipleLocator(1)) + ax.set_ylim(0, 1) + ax.grid() + # + pdf.savefig(fig, bbox_inches="tight") + plt.close() + # AR per-petal, per-tracer n(z) + for tracer in tracers: + faprgrm, mask, dtkey, xlim, ylim = get_tracer_props(tracer) + istracer = ds[faprgrm]["SURVEY"] == survey + istracer &= (ds[faprgrm][dtkey] & mask[tracer]) > 0 + istracer &= ds[faprgrm]["VALID"] + istracer_zok = (istracer) & (ds[faprgrm]["ZOK"]) + bins = np.arange(xlim[0], xlim[1] + 0.05, 0.05) + # + if ntiles_surv[faprgrm] > 0: + fig = plt.figure(figsize=(40, 5)) + gs = gridspec.GridSpec(1, 10, wspace=0.3) + for petal in petals: + ax = plt.subplot(gs[petal]) + _ = ax.hist( + ds[faprgrm]["Z"][istracer_zok], + bins=bins, + density=True, + histtype="stepfilled", + alpha=0.5, + color=colors[tracer], + label="{} All petals".format(tracer), ) - ) - ax.set_xlabel("Z") - if petal == 0: - ax.set_ylabel("Normalized counts") - else: - ax.set_yticklabels([]) - ax.set_xlim(xlim) - ax.set_ylim(ylim) - ax.grid() - ax.set_axisbelow(True) - ax.legend(loc=1) - ax.text( - 0.97, 0.8, - "DELTACHI2 > {}".format(dchi2_threshold), - fontsize=10, fontweight="bold", color="k", - ha="right", transform=ax.transAxes, - ) - pdf.savefig(fig, bbox_inches="tight") - plt.close() + _ = ax.hist( + ds[faprgrm]["Z"][(istracer_zok) & (ds[faprgrm]["PETAL_LOC"] == petal)], + bins=bins, + density=True, + histtype="step", + alpha=1.0, + color="k", + label="{} PETAL_LOC = {}".format(tracer, petal), + ) + ax.set_title( + "{} {}-{} tiles from {}".format( + ntiles_surv[faprgrm], + survey.upper(), + faprgrm.upper(), + night, + ) + ) + ax.set_xlabel("Z") + if petal == 0: + ax.set_ylabel("Normalized counts") + else: + ax.set_yticklabels([]) + ax.set_xlim(xlim) + ax.set_ylim(ylim) + ax.grid() + ax.set_axisbelow(True) + ax.legend(loc=1) + ax.text( + 0.97, 0.8, + "DELTACHI2 > {}".format(dchi2_threshold), + fontsize=10, fontweight="bold", color="k", + ha="right", transform=ax.transAxes, + ) + pdf.savefig(fig, bbox_inches="tight") + plt.close() def path_full2web(fn): From fa2388036b32147beb2403e9fb539e839f13def4 Mon Sep 17 00:00:00 2001 From: anand_raichoor Date: Thu, 16 Dec 2021 21:48:58 -0800 Subject: [PATCH 12/13] simpler syntax to check header keywords --- py/desispec/night_qa.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/py/desispec/night_qa.py b/py/desispec/night_qa.py index 429d91259..b7e1e2342 100644 --- a/py/desispec/night_qa.py +++ b/py/desispec/night_qa.py @@ -100,7 +100,7 @@ def get_surveys_night_expids( fafns = glob(os.path.join(os.path.dirname(fns[i]), "fiberassign-??????.fits*")) if len(fafns) > 0: fahdr = fits.getheader(fafns[0], 0) - if "SURVEY" in [cards[0] for cards in fahdr.cards]: + if "SURVEY" in fahdr: survey = fahdr["SURVEY"] else: survey = fahdr["FA_SURV"] @@ -803,7 +803,7 @@ def create_petalnz_pdf(outpdf, night, prod, tileids, surveys, dchi2_threshold=25 log.warning("no {} file, proceeding to next tile".format(fn)) continue hdr = fits.getheader(fn, "FIBERQA") - if "FAPRGRM" not in [cards[0] for cards in hdr.cards]: + if "FAPRGRM" not in hdr: log.warning("no FAPRGRM in {} header, proceeding to next tile".format(fn)) continue faprgrm = hdr["FAPRGRM"].lower() From c6d1fcfb53d4e39ef6bdfbb3db474111e9a7470f Mon Sep 17 00:00:00 2001 From: anand_raichoor Date: Thu, 16 Dec 2021 21:51:34 -0800 Subject: [PATCH 13/13] remove unused leftover --- py/desispec/night_qa.py | 1 - 1 file changed, 1 deletion(-) diff --git a/py/desispec/night_qa.py b/py/desispec/night_qa.py index b7e1e2342..a92e6f72b 100644 --- a/py/desispec/night_qa.py +++ b/py/desispec/night_qa.py @@ -111,7 +111,6 @@ def get_surveys_night_expids( tileids.append(hdr["TILEID"]) surveys.append(survey) expids, tileids, surveys = np.array(expids), np.array(tileids), np.array(surveys) - msg = "for NIGHT={} found" per_surv = [] for survey in np.unique(surveys): sel = surveys == survey