diff --git a/bin/select_secondary b/bin/select_secondary index 7a8c2756a..16de05f0e 100755 --- a/bin/select_secondary +++ b/bin/select_secondary @@ -43,7 +43,7 @@ hdr = fitsio.read_header(fns[0], 'SCND_TARG') surv = hdr["SURVEY"].rstrip() # ADM find the SCND_DIR environment variable, if it wasn't passed. -scxdir = _get_scxdir(ns.scnddir) +scxdir = _get_scxdir(ns.scnddir, survey=surv) # ADM and augment the scxdir if this is an SV set of primary files. if surv != 'main': scxdir = os.path.join(scxdir, surv) diff --git a/bin/select_sv_targets b/bin/select_sv_targets index ee57153f9..b36602442 100755 --- a/bin/select_sv_targets +++ b/bin/select_sv_targets @@ -67,7 +67,7 @@ ap.add_argument("--nomaskbits", action='store_true', ap.add_argument("--writeall", action='store_true', help="Default behavior is to split targets by bright/dark-time surveys. Send this to ALSO write a file of ALL targets") ap.add_argument("-nos", "--nosecondary", action='store_true', - help="Do NOT create TARGETID look-up files for secondary targets in $SCNDIR/outdata/priminfo-drversion-desitargetversion/$dest.fits (where $dest is the basename of dest)") + help="Do NOT create TARGETID look-up files for secondary targets in $SCNDIR/outdata/desitargetversion/priminfo-drversion-desitargetversion/$dest.fits (where $dest is the basename of dest)") ap.add_argument("--scnddir", help="Base directory of secondary target files (e.g. '/project/projectdirs/desi/target/secondary' at NERSC). "+ "Defaults to SCND_DIR environment variable. Not needed if --nosecondary is sent.") @@ -80,13 +80,14 @@ ns = ap.parse_args() # ADM build the list of command line arguments as # ADM bundlefiles potentially needs to know about them. extra = " --numproc {}".format(ns.numproc) -if ns.tcnames is not None: - extra += " --tcnames {}".format(ns.tcnames) nsdict = vars(ns) -for nskey in ["noresolve", "nomaskbits", "writeall", - "nosecondary", "nobackup", "nochecksum"]: - if nsdict[nskey]: - extra += " --{}".format(nskey) +for nskey in ["tcnames", "noresolve", "nomaskbits", "writeall", + "nosecondary", "nobackup", "nochecksum", "scnddir"]: + if isinstance(nsdict[nskey], bool): + if nsdict[nskey]: + extra += " --{}".format(nskey) + elif nsdict[nskey] is not None: + extra += " --{} {}".format(nskey, nsdict[nskey]) infiles = io.list_sweepfiles(ns.sweepdir) if ns.sweepdir2 is not None: @@ -148,25 +149,29 @@ targets, infn = select_targets( tcnames=tcnames, survey=survey, backup=not(ns.nobackup), resolvetargs=not(ns.noresolve), mask=not(ns.nomaskbits), return_infiles=True ) -# ADM Set the list of infiles actually processed by select_targets() to -# ADM None if we DON'T want to write their checksums to the output file. -if ns.nochecksum: - shatab = None -else: - shatab = get_checksums(infn, verbose=True) if ns.bundlefiles is None: + # ADM Set the list of infiles actually processed by select_targets() to + # ADM None if we DON'T want to write their checksums to the output file. + if ns.nochecksum: + shatab = None + else: + shatab = get_checksums(infn, verbose=True) + # ADM only run secondary functions if --nosecondary was not passed. scndout = None if not ns.nosecondary and len(targets) > 0: from desitarget.secondary import _get_scxdir, match_secondary # ADM read secondary target directory. - scxdir = _get_scxdir(ns.scnddir) + scxdir = _get_scxdir(ns.scnddir, survey=survey) # ADM construct a label for the secondary file for TARGETID look-ups. - # ADM use RELEASE to determine the release string for the targets. - drint = np.max(targets['RELEASE']//1000) + try: + drint = int(ns.sweepdir.split("dr")[1][0]) + except (ValueError, IndexError, AttributeError): + drint = "X" scndoutdn = "priminfo-dr{}-{}".format(drint, desitarget_version) - scndoutdn = os.path.join(scxdir, survey, "outdata", scndoutdn) + scndoutdn = os.path.join(scxdir, survey, + "outdata", desitarget_version, scndoutdn) if not os.path.exists(scndoutdn): log.info("making directory...{}".format(scndoutdn)) os.makedirs(scndoutdn) @@ -181,7 +186,7 @@ if ns.bundlefiles is None: scndout = os.path.join(scndoutdn, scndoutfn) log.info("writing files of primary matches to...{}".format(scndout)) targets = match_secondary(targets, scxdir, scndout, sep=1., - pix=pixlist, nside=ns.nside) + pix=pixlist, nside=ns.nside, swfiles=infn) if ns.mask: targets = mask_targets(targets, inmaskfile=ns.mask, nside=nside) diff --git a/bin/select_targets b/bin/select_targets index 66c0ed7e8..0e5f0aa90 100755 --- a/bin/select_targets +++ b/bin/select_targets @@ -69,7 +69,7 @@ ap.add_argument("--nomaskbits", action='store_true', ap.add_argument("--writeall", action='store_true', help="Default behavior is to split targets by bright/dark-time surveys. Send this to ALSO write a file of ALL targets") ap.add_argument("-nos", "--nosecondary", action='store_true', - help="Do NOT create TARGETID look-up files for secondary targets in $SCNDIR/outdata/priminfo-drversion-desitargetversion/$dest.fits (where $dest is the basename of dest)") + help="Do NOT create TARGETID look-up files for secondary targets in $SCNDIR/outdata/desitargetversion/priminfo-drversion-desitargetversion/$dest.fits (where $dest is the basename of dest)") ap.add_argument("--scnddir", help="Base directory of secondary target files (e.g. '/project/projectdirs/desi/target/secondary' at NERSC). "+ "Defaults to SCND_DIR environment variable. Not needed if --nosecondary is sent.") @@ -82,13 +82,14 @@ ns = ap.parse_args() # ADM build the list of command line arguments as # ADM bundlefiles potentially needs to know about them. extra = " --numproc {}".format(ns.numproc) -if ns.tcnames is not None: - extra += " --tcnames {}".format(ns.tcnames) nsdict = vars(ns) -for nskey in ["noresolve", "nomaskbits", "writeall", - "nosecondary", "nobackup", "nochecksum"]: - if nsdict[nskey]: - extra += " --{}".format(nskey) +for nskey in ["tcnames", "noresolve", "nomaskbits", "writeall", + "nosecondary", "nobackup", "nochecksum", "scnddir"]: + if isinstance(nsdict[nskey], bool): + if nsdict[nskey]: + extra += " --{}".format(nskey) + elif nsdict[nskey] is not None: + extra += " --{} {}".format(nskey, nsdict[nskey]) infiles = io.list_sweepfiles(ns.sweepdir) if ns.sweepdir2 is not None: @@ -149,14 +150,15 @@ targets, infn = select_targets( tcnames=tcnames, survey='main', backup=not(ns.nobackup), resolvetargs=not(ns.noresolve), mask=not(ns.nomaskbits), return_infiles=True ) -# ADM Set the list of infiles actually processed by select_targets() to -# ADM None if we DON'T want to write their checksums to the output file. -if ns.nochecksum: - shatab = None -else: - shatab = get_checksums(infn, verbose=True) if ns.bundlefiles is None: + # ADM Set the list of infiles actually processed by select_targets() to + # ADM None if we DON'T want to write their checksums to the output file. + if ns.nochecksum: + shatab = None + else: + shatab = get_checksums(infn, verbose=True) + # ADM only run secondary functions if --nosecondary was not passed. scndout = None if not ns.nosecondary and len(targets) > 0: @@ -164,10 +166,13 @@ if ns.bundlefiles is None: # ADM read secondary target directory. scxdir = _get_scxdir(ns.scnddir) # ADM construct a label for the secondary file for TARGETID look-ups. - # ADM use RELEASE to determine the release string for the targets. - drint = np.max(targets['RELEASE']//1000) + try: + drint = int(ns.sweepdir.split("dr")[1][0]) + except (ValueError, IndexError, AttributeError): + drint = "X" scndoutdn = "priminfo-dr{}-{}".format(drint, desitarget_version) - scndoutdn = os.path.join(scxdir, "outdata", scndoutdn) + scndoutdn = os.path.join(scxdir, + "outdata", desitarget_version, scndoutdn) if not os.path.exists(scndoutdn): log.info("making directory...{}".format(scndoutdn)) os.makedirs(scndoutdn) @@ -182,7 +187,7 @@ if ns.bundlefiles is None: scndout = os.path.join(scndoutdn, scndoutfn) log.info("writing files of primary matches to...{}".format(scndout)) targets = match_secondary(targets, scxdir, scndout, sep=1., - pix=pixlist, nside=ns.nside) + pix=pixlist, nside=ns.nside, swfiles=infn) if ns.mask: targets = mask_targets(targets, inmaskfile=ns.mask, nside=nside) diff --git a/doc/changes.rst b/doc/changes.rst index a8fe9e348..e672d991d 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -5,6 +5,14 @@ desitarget Change Log 0.47.1 (unreleased) ------------------- +* First run of secondaries with real target files [`PR #669`_]. Includes: + * Add Gaia-only standard stars to the MWS masks for SV, Main Survey: + * `GAIA_STD_FAINT`, `GAIA_STD_BRIGHT`, `GAIA_STD_WD`. + * General optimization, updating and debugging of the secondary code. + * Get `TARGETIDs` from the input sweeps, not just the actual targets. + * Add the first full bitmask for the SV1 secondary target files. + * Updates to the data model to better reflect the primary targets. +* Clean-up minor style and doc issues from `PR #636`_ [`PR #668`_]. * Updates and bug fixes for DR9 now SV is on-sky [`PR #665`_]. Includes: * Pass `MASKBITS` column forward for GFAs. * Bug fixes necessitated by target files having a second extension. @@ -15,12 +23,12 @@ desitarget Change Log * Simplify stellar SV bits [`PR #636`_]: * Secondary bit requirement for main stellar SV program to 4 bits. * Primary bright science WDs use the old algorithmic selection. -* Clean-up minor style and doc issues from `PR #636`_ [`PR #668`_]. .. _`PR #636`: https://github.com/desihub/desitarget/pull/636 .. _`PR #664`: https://github.com/desihub/desitarget/pull/664 .. _`PR #665`: https://github.com/desihub/desitarget/pull/665 .. _`PR #668`: https://github.com/desihub/desitarget/pull/668 +.. _`PR #669`: https://github.com/desihub/desitarget/pull/669 0.47.0 (2020-12-10) ------------------- diff --git a/py/desitarget/cuts.py b/py/desitarget/cuts.py index 61805125a..750d2655c 100644 --- a/py/desitarget/cuts.py +++ b/py/desitarget/cuts.py @@ -19,17 +19,20 @@ import numbers import sys +import fitsio import numpy as np import healpy as hp from pkg_resources import resource_filename import numpy.lib.recfunctions as rfn from importlib import import_module +import astropy.units as u +from astropy.coordinates import SkyCoord from astropy.table import Table, Row from desitarget import io from desitarget.internal import sharedmem -from desitarget.gaiamatch import match_gaia_to_primary +from desitarget.gaiamatch import match_gaia_to_primary, find_gaia_files_hp from desitarget.gaiamatch import pop_gaia_coords, pop_gaia_columns from desitarget.gaiamatch import gaia_dr_from_ref_cat, is_in_Galaxy from desitarget.targets import finalize, resolve @@ -138,6 +141,129 @@ def shift_photo_north(gflux=None, rflux=None, zflux=None): return gshift, rshift, zshift +def isGAIA_STD(ra=None, dec=None, galb=None, gaiaaen=None, pmra=None, pmdec=None, + parallax=None, parallaxovererror=None, gaiabprpfactor=None, + gaiasigma5dmax=None, gaiagmag=None, gaiabmag=None, gaiarmag=None, + gaiadupsource=None, gaiaparamssolved=None, + primary=None, test=False, nside=2): + """Standards based solely on Gaia data. + + Parameters + ---------- + test : :class:`bool`, optional, defaults to ``False`` + If ``True``, then we're running unit tests and don't have to + find and read every possible Gaia file. + nside : :class:`int`, optional, defaults to 2 + (NESTED) HEALPix nside, if targets are being parallelized. + The default of 2 should be benign for serial processing. + + Returns + ------- + :class:`array_like` + ``True`` if the object is a bright "GAIA_STD_FAINT" target. + :class:`array_like` + ``True`` if the object is a faint "GAIA_STD_BRIGHT" target. + :class:`array_like` + ``True`` if the object is a white dwarf "GAIA_STD_WD" target. + + Notes + ----- + - Current version (01/08/21) is version XXX on `the wiki`_. + - See :func:`~desitarget.cuts.set_target_bits` for other parameters. + """ + if primary is None: + primary = np.ones_like(gaiagmag, dtype='?') + + # ADM restrict all classes to dec >= -30. + primary &= dec >= -30. + std = primary.copy() + + # ADM the regular "standards" codes need to know whether something has + # ADM a Gaia match. Here, everything is a Gaia match. + gaia = np.ones_like(gaiagmag, dtype='?') + + # ADM determine the Gaia-based white dwarf standards. + std_wd = isMWS_WD( + primary=primary, gaia=gaia, galb=galb, astrometricexcessnoise=gaiaaen, + pmra=pmra, pmdec=pmdec, parallax=parallax, + parallaxovererror=parallaxovererror, photbprpexcessfactor=gaiabprpfactor, + astrometricsigma5dmax=gaiasigma5dmax, gaiagmag=gaiagmag, + gaiabmag=gaiabmag, gaiarmag=gaiarmag + ) + + # ADM apply the Gaia quality cuts for standards. + std &= isSTD_gaia(primary=primary, gaia=gaia, astrometricexcessnoise=gaiaaen, + pmra=pmra, pmdec=pmdec, parallax=parallax, + dupsource=gaiadupsource, paramssolved=gaiaparamssolved, + gaiagmag=gaiagmag, gaiabmag=gaiabmag, gaiarmag=gaiarmag) + + # ADM restrict to point sources. + ispsf = np.logical_or( + (gaiagmag <= 19.) * (gaiaaen < 10.**0.5), + (gaiagmag >= 19.) * (gaiaaen < 10.**(0.5 + 0.2*(gaiagmag - 19.))) + ) + std &= ispsf + + # ADM apply the Gaia color cuts for standards. + bprp = gaiabmag - gaiarmag + gbp = gaiagmag - gaiabmag + std &= bprp > 0.2 + std &= bprp < 0.9 + std &= gbp > -1.*bprp/2.0 + std &= gbp < 0.3-bprp/2.0 + + # ADM remove any sources that have neighbors in Gaia within 3.5"... + # ADM for speed, run only sources for which std is still True. + log.info("Isolating Gaia-only standards...t={:.1f}s".format(time()-start)) + ii_true = np.where(std)[0] + if len(ii_true) > 0: + # ADM determine the pixels of interest. + theta, phi = np.radians(90-dec), np.radians(ra) + pixlist = list(set(hp.ang2pix(nside, theta, phi, nest=True))) + # ADM read in the necessary Gaia files. + fns = find_gaia_files_hp(nside, pixlist, neighbors=True) + gaiaobjs = [] + gaiacols = ["RA", "DEC", "PHOT_G_MEAN_MAG", "PHOT_RP_MEAN_MAG"] + for i, fn in enumerate(fns): + if i % 25 == 0: + log.info("Read {}/{} files for Gaia-only standards...t={:.1f}s" + .format(i, len(fns), time()-start)) + try: + gaiaobjs.append(fitsio.read(fn, columns=gaiacols)) + except OSError: + if test: + pass + else: + msg = "failed to find or open the following file: (ffopen) " + msg += fn + log.critical(msg) + raise OSError + gaiaobjs = np.concatenate(gaiaobjs) + # ADM match the standards to the broader Gaia sources at 3.5". + matchrad = 3.5*u.arcsec + cstd = SkyCoord(ra[ii_true]*u.degree, dec[ii_true]*u.degree) + cgaia = SkyCoord(gaiaobjs["RA"]*u.degree, gaiaobjs["DEC"]*u.degree) + idstd, idgaia, d2d, _ = cgaia.search_around_sky(cstd, matchrad) + # ADM remove source matches with d2d=0 (i.e. the source itself!). + idgaia, idstd = idgaia[d2d > 0], idstd[d2d > 0] + # ADM remove matches within 5 mags of a Gaia source. + badmag = ( + (gaiagmag[ii_true][idstd] + 5 > gaiaobjs["PHOT_G_MEAN_MAG"][idgaia]) | + (gaiarmag[ii_true][idstd] + 5 > gaiaobjs["PHOT_RP_MEAN_MAG"][idgaia])) + std[ii_true[idstd][badmag]] = False + + # ADM add the brightness cuts in Gaia G-band. + std_bright = std.copy() + std_bright &= gaiagmag >= 15 + std_bright &= gaiagmag < 18 + + std_faint = std.copy() + std_faint &= gaiagmag >= 16 + std_faint &= gaiagmag < 19 + + return std_faint, std_bright, std_wd + + def isBACKUP(ra=None, dec=None, gaiagmag=None, primary=None): """BACKUP targets based on Gaia magnitudes. @@ -367,7 +493,7 @@ def notinELG_mask(maskbits=None, gsnr=None, rsnr=None, zsnr=None, def isELG_colors(gflux=None, rflux=None, zflux=None, w1flux=None, w2flux=None, south=True, primary=None): """Color cuts for ELG target selection classes - (see, e.g., :func:`desitarget.cuts.set_target_bits` for parameters). + (see, e.g., :func:`~desitarget.cuts.set_target_bits` for parameters). """ if primary is None: primary = np.ones_like(rflux, dtype='?') @@ -2168,7 +2294,8 @@ def set_target_bits(photsys_north, photsys_south, obs_rflux, return desi_target, bgs_target, mws_target -def apply_cuts_gaia(numproc=4, survey='main', nside=None, pixlist=None): +def apply_cuts_gaia(numproc=4, survey='main', nside=None, pixlist=None, + test=False): """Gaia-only-based target selection, return target mask arrays. Parameters @@ -2185,6 +2312,10 @@ def apply_cuts_gaia(numproc=4, survey='main', nside=None, pixlist=None): Only return targets in a set of (NESTED) HEALpixels at `nside`. Useful for parallelizing, as input files will only be processed if they touch a pixel in the passed list. + test : :class:`bool`, optional, defaults to ``False`` + If ``True``, then we're running unit tests and don't have to find + and read every possible Gaia file when calling + :func:`~desitarget.cuts.apply_cuts_gaia`. Returns ------- @@ -2233,27 +2364,41 @@ def apply_cuts_gaia(numproc=4, survey='main', nside=None, pixlist=None): # ADM or are north of dec=-30. gaiaobjs = all_gaia_in_tiles(maglim=19, numproc=numproc, allsky=True, mindec=-30, mingalb=0, addobjid=True, - nside=nside, pixlist=pixlist) + nside=nside, pixlist=pixlist, addparams=True) # ADM the convenience function we use adds an empty TARGETID # ADM field which we need to remove before finalizing. gaiaobjs = rfn.drop_fields(gaiaobjs, "TARGETID") - primary = np.ones_like(gaiaobjs, dtype=bool) - # ADM the relevant input quantities. ra = gaiaobjs["RA"] dec = gaiaobjs["DEC"] - gaiagmag = gaiaobjs["GAIA_PHOT_G_MEAN_MAG"] + gaia, pmra, pmdec, parallax, parallaxovererror, parallaxerr, gaiagmag, gaiabmag, \ + gaiarmag, gaiaaen, gaiadupsource, Grr, gaiaparamssolved, gaiabprpfactor, \ + gaiasigma5dmax, galb = _prepare_gaia(gaiaobjs) # ADM determine if an object is a BACKUP target. + primary = np.ones_like(gaiaobjs, dtype=bool) backup_bright, backup_faint, backup_very_faint = targcuts.isBACKUP( ra=ra, dec=dec, gaiagmag=gaiagmag, primary=primary ) + # ADM determine if a target is a Gaia-only standard. + primary = np.ones_like(gaiaobjs, dtype=bool) + std_faint, std_bright, std_wd = targcuts.isGAIA_STD( + ra=ra, dec=dec, galb=galb, gaiaaen=gaiaaen, pmra=pmra, pmdec=pmdec, + parallax=parallax, parallaxovererror=parallaxovererror, + gaiabprpfactor=gaiabprpfactor, gaiasigma5dmax=gaiasigma5dmax, + gaiagmag=gaiagmag, gaiabmag=gaiabmag, gaiarmag=gaiarmag, + gaiadupsource=gaiadupsource, gaiaparamssolved=gaiaparamssolved, + primary=primary, nside=nside, test=test) + # ADM Construct the target flag bits. mws_target = backup_bright * mws_mask.BACKUP_BRIGHT mws_target |= backup_faint * mws_mask.BACKUP_FAINT mws_target |= backup_very_faint * mws_mask.BACKUP_VERY_FAINT + mws_target |= std_faint * mws_mask.GAIA_STD_FAINT + mws_target |= std_bright * mws_mask.GAIA_STD_BRIGHT + mws_target |= std_wd * mws_mask.GAIA_STD_WD bgs_target = np.zeros_like(mws_target) @@ -2408,7 +2553,7 @@ def select_targets(infiles, numproc=4, qso_selection='randomforest', extra=None, radecbox=None, radecrad=None, mask=True, tcnames=["ELG", "QSO", "LRG", "MWS", "BGS", "STD"], survey='main', resolvetargs=True, backup=True, - return_infiles=False): + return_infiles=False, test=False): """Process input files in parallel to select targets. Parameters @@ -2464,6 +2609,10 @@ def select_targets(infiles, numproc=4, qso_selection='randomforest', If ``True``, also return the actual files from `infile` processed. Useful when running with `pixlist`, `radecbox` or `radecrad` to see which files were actually required. + test : :class:`bool`, optional, defaults to ``False`` + If ``True``, then we're running unit tests and don't have to find + and read every possible Gaia file when calling + :func:`~desitarget.cuts.apply_cuts_gaia`. Returns ------- @@ -2632,7 +2781,7 @@ def _update_status(result): # ADM set the target bits that are based only on Gaia. gaia_desi_target, gaia_bgs_target, gaia_mws_target, gaiaobjs = \ apply_cuts_gaia(numproc=numproc4, survey=survey, nside=nside, - pixlist=pixlist) + pixlist=pixlist, test=test) # ADM it's possible that somebody could pass HEALPixels that # ADM contain no additional targets. diff --git a/py/desitarget/data/targetmask.yaml b/py/desitarget/data/targetmask.yaml index 21d19a02f..61b5daee6 100644 --- a/py/desitarget/data/targetmask.yaml +++ b/py/desitarget/data/targetmask.yaml @@ -101,6 +101,11 @@ mws_mask: - [MWS_MAIN_RED_NORTH, 12, "MWS magnitude limited red sample tuned for Bok/Mosaic", {obsconditions: BRIGHT}] - [MWS_MAIN_RED_SOUTH, 13, "MWS magnitude limited red sample tuned for DECam", {obsconditions: BRIGHT}] + # ADM Standard stars based only on Gaia. + - [GAIA_STD_FAINT, 33, "Standard stars for dark/gray conditions", {obsconditions: DARK|GRAY}] + - [GAIA_STD_WD, 34, "White Dwarf stars", {obsconditions: DARK|GRAY|BRIGHT}] + - [GAIA_STD_BRIGHT, 35, "Standard stars for BRIGHT conditions", {obsconditions: BRIGHT}] + # ADM back-up targets for poor conditions and as filler. - [BACKUP_BRIGHT, 60, "Bright backup Gaia targets", {obsconditions: DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18}] - [BACKUP_FAINT, 61, "Fainter backup Gaia targets", {obsconditions: DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18}] @@ -231,6 +236,10 @@ priorities: BACKUP_BRIGHT: {UNOBS: 9, DONE: 9, OBS: 9, DONOTOBSERVE: 0} BACKUP_FAINT: {UNOBS: 8, DONE: 8, OBS: 8, DONOTOBSERVE: 0} BACKUP_VERY_FAINT: {UNOBS: 7, DONE: 7, OBS: 7, DONOTOBSERVE: 0} + # ADM Standards are special; priorities don't apply. + GAIA_STD_FAINT: -1 + GAIA_STD_WD: -1 + GAIA_STD_BRIGHT: -1 # ADM secondary target priorities. Probably all have very low UNOBS... scnd_mask: @@ -305,6 +314,10 @@ numobs: BACKUP_BRIGHT: 1 BACKUP_FAINT: SAME_AS_BACKUP_BRIGHT BACKUP_VERY_FAINT: SAME_AS_BACKUP_BRIGHT + # ADM Standards are special; numobs doesn't apply. + GAIA_STD_FAINT: -1 + GAIA_STD_WD: -1 + GAIA_STD_BRIGHT: -1 # ADM initial number of observations for secondary targets scnd_mask: diff --git a/py/desitarget/io.py b/py/desitarget/io.py index d01f0f5fa..f92372536 100644 --- a/py/desitarget/io.py +++ b/py/desitarget/io.py @@ -864,7 +864,7 @@ def write_secondary(targdir, data, primhdr=None, scxdir=None, obscon=None, header of the output `filename`. obscon : :class:`str`, optional, defaults to `None` Can pass one of "DARK" or "BRIGHT". If passed, don't write the - full set of secondary target that do not match a primary, + full set of secondary targets that do not match a primary, rather only write targets appropriate for "DARK|GRAY" or "BRIGHT" observing conditions. The relevant `PRIORITY_INIT` and `NUMOBS_INIT` columns will be derived from @@ -886,13 +886,13 @@ def write_secondary(targdir, data, primhdr=None, scxdir=None, obscon=None, Two sets of files are written: - The file of secondary targets that do not match a primary target is written to `targdir`. Such secondary targets - are determined from having `RELEASE==0` and `SKY==0` - in the `TARGETID`. Only targets with `PRIORITY_INIT > -1` - are written to this file (this allows duplicates to be - resolved in, e.g., :func:`~desitarget.secondary.finalize()` + are determined from having "PRIM_MATCH"=``False`` in `data`. + Only targets with `PRIORITY_INIT > -1` are written to this file + (this allows duplicates to be resolved in, e.g., + :func:`~desitarget.secondary.finalize()`. - Each secondary target that, presumably, was initially drawn from the "indata" subdirectory of `scxdir` is written to - an "outdata/targdir" subdirectory of `scxdir`. + the "outdata" subdirectory of `scxdir`. """ # ADM grab the scxdir, it it wasn't passed. from desitarget.secondary import _get_scxdir @@ -924,9 +924,13 @@ def write_secondary(targdir, data, primhdr=None, scxdir=None, obscon=None, np.random.seed(616) data["SUBPRIORITY"] = np.random.random(ntargs) - # ADM remove the SCND_TARGET_INIT and SCND_ORDER columns. - scnd_target_init, scnd_order = data["SCND_TARGET_INIT"], data["SCND_ORDER"] - data = rfn.drop_fields(data, ["SCND_TARGET_INIT", "SCND_ORDER"]) + # ADM remove the SCND_TARGET_INIT, SCND_ORDER and PRIM_MATCH columns. + scnd_target_init = data["SCND_TARGET_INIT"] + scnd_order = data["SCND_ORDER"] + prim_match = data["PRIM_MATCH"] + + data = rfn.drop_fields(data, + ["SCND_TARGET_INIT", "SCND_ORDER", "PRIM_MATCH"]) # ADM we only need a subset of the columns where we match a primary. smalldata = rfn.drop_fields(data, ["PRIORITY_INIT", "SUBPRIORITY", "NUMOBS_INIT", "OBSCONDITIONS"]) @@ -937,14 +941,15 @@ def write_secondary(targdir, data, primhdr=None, scxdir=None, obscon=None, scnd_mask = mx[3] # ADM construct the output full and reduced file name. - filename = find_target_files(targdir, dr=drint, flavor="targets", - survey=survey, obscon=obscon, nohp=True) - filenam = os.path.splitext(os.path.basename(filename))[0] + filename = find_target_files(targdir, dr=drint, flavor="targets", nohp=True, + survey=survey, obscon=obscon, resolve=None) # ADM write out the file of matches for every secondary bit. - scxoutdir = os.path.join(scxdir, 'outdata', filenam) + scxoutdir = os.path.join(scxdir, 'outdata', desitarget_version) if obscon is not None: scxoutdir = os.path.join(scxoutdir, obscon.lower()) + else: + scxoutdir = os.path.join(scxoutdir, "no-obscon") os.makedirs(scxoutdir, exist_ok=True) # ADM and write out the information for each bit. @@ -968,9 +973,8 @@ def write_secondary(targdir, data, primhdr=None, scxdir=None, obscon=None, os.makedirs(os.path.dirname(filename), exist_ok=True) # ADM standalone secondaries have PRIORITY_INIT > -1 and - # ADM release before DR1 (release < 1000). - objid, brickid, release, mock, sky, gaiadr = decode_targetid(data["TARGETID"]) - ii = (release < 1000) & (data["PRIORITY_INIT"] > -1) + # ADM don't have PRIM_MATCH set. + ii = ~prim_match & (data["PRIORITY_INIT"] > -1) # ADM ...write them out. write_with_units(filename, data[ii], extname='SCND_TARGETS', header=hdr) @@ -2162,6 +2166,7 @@ def find_target_files(targdir, dr='X', flavor="targets", survey="main", resolve : :class:`bool`, optional, defaults to ``True`` If ``True`` then find the `resolve` file. Otherwise find the `noresolve` file. Relevant if `flavor` is `targets` or `randoms`. + Pass ``None`` to substitute `resolve` with "secondary". supp : :class:`bool`, optional, defaults to ``False`` If ``True`` then find the supplemental targets file. Overrides the `obscon` option. @@ -2219,8 +2224,11 @@ def find_target_files(targdir, dr='X', flavor="targets", survey="main", log.critical(msg) raise ValueError(msg) res = "noresolve" - if resolve: - res = "resolve" + if resolve is None: + res = "secondary" + else: + if resolve: + res = "resolve" resdir = "" if flavor in ["targets", "randoms"]: resdir = res @@ -2401,7 +2409,8 @@ def read_target_files(filename, columns=None, rows=None, header=False, """ start = time() # ADM start with some checking that this is a target file. - targtypes = "TARGETS", "GFA_TARGETS", "SKY_TARGETS", "MASKS", "MTL" + targtypes = ["TARGETS", "GFA_TARGETS", "SKY_TARGETS", + "MASKS", "MTL", "SCND_TARGETS"] # ADM read in the FITS extension info. f = fitsio.FITS(filename) if len(f) != 2: diff --git a/py/desitarget/secondary.py b/py/desitarget/secondary.py index d6ee164c4..a35cdba16 100644 --- a/py/desitarget/secondary.py +++ b/py/desitarget/secondary.py @@ -55,8 +55,9 @@ from desitarget.internal import sharedmem from desitarget.geomask import radec_match_to, add_hp_neighbors, is_in_hp +from desitarget.gaiamatch import gaiadatamodel -from desitarget.targets import encode_targetid, main_cmx_or_sv +from desitarget.targets import encode_targetid, main_cmx_or_sv, resolve from desitarget.targets import set_obsconditions, initial_priority_numobs from desitarget.targetmask import obsconditions @@ -81,18 +82,23 @@ # or svX/data/svX_targetmask.yaml (scnd_mask). # ADM Note that TARGETID for secondary-only targets is unique because # ADM RELEASE is < 1000 (before DR1) for secondary-only targets. +# ADM also add needed columns for fiberassign from the Gaia data model. +gaiacols = ["PARALLAX", "GAIA_PHOT_G_MEAN_MAG", 'GAIA_ASTROMETRIC_EXCESS_NOISE'] +gaiadt = [(gaiadatamodel[gaiacols].dtype.names[i], + gaiadatamodel[gaiacols].dtype[i].str) for i in range(len(gaiacols))] + outdatamodel = np.array([], dtype=[ ('RA', '>f8'), ('DEC', '>f8'), ('PMRA', '>f4'), ('PMDEC', '>f4'), - ('REF_EPOCH', '>f4'), ('OVERRIDE', '?'), + ('REF_EPOCH', '>f4'), ('OVERRIDE', '?')] + gaiadt + [ ('TARGETID', '>i8'), ('DESI_TARGET', '>i8'), ('SCND_TARGET', '>i8'), ('PRIORITY_INIT', '>i8'), ('SUBPRIORITY', '>f8'), - ('NUMOBS_INIT', '>i8'), ('OBSCONDITIONS', '>i8') -]) + ('NUMOBS_INIT', '>i8'), ('OBSCONDITIONS', '>i8')]) # ADM extra columns that are used during processing but are # ADM not an official part of the input or output data model. +# ADM PRIM_MATCH records whether a secondary matches a primary TARGET. suppdatamodel = np.array([], dtype=[ - ('SCND_TARGET_INIT', '>i8'), ('SCND_ORDER', '>i4') + ('SCND_TARGET_INIT', '>i8'), ('SCND_ORDER', '>i4'), ('PRIM_MATCH', '?') ]) @@ -126,7 +132,7 @@ def duplicates(seq): return ((key, np.array(locs)) for key, locs in tally.items() if len(locs) > 1) -def _get_scxdir(scxdir=None): +def _get_scxdir(scxdir=None, survey=""): """Retrieve the base secondary directory with error checking. Parameters @@ -135,6 +141,9 @@ def _get_scxdir(scxdir=None): Directory containing secondary target files to which to match. If not specified, the directory is taken to be the value of the :envvar:`SCND_DIR` environment variable. + survey : :class:`str`, optional, defaults to "" for the Main Survey. + Flavor of survey that we're processing, e.g., "sv1". Don't pass + anything for "main" (the Main Survey). Returns ------- @@ -154,7 +163,8 @@ def _get_scxdir(scxdir=None): # ADM also fail if the indata, outdata and docs directories don't # ADM exist in the secondary directory. - for subdir in "docs", "indata", "outdata": + checkdir = [os.path.join(survey, d) for d in ["docs", "indata", "outdata"]] + for subdir in checkdir: if not os.path.isdir(os.path.join(scxdir, subdir)): msg = '{} directory not found in {}'.format(subdir, scxdir) log.critical(msg) @@ -200,8 +210,9 @@ def _check_files(scxdir, scnd_mask): # ADM extensions in each of the indata and docs directories. setdic = {} for subdir in 'indata', 'docs': - # ADM retrieve the full file names. - fnswext = os.listdir(os.path.join(scxdir, subdir)) + # ADM retrieve the full file names. Ignore directories. + fnswext = [fn for fn in os.listdir(os.path.join(scxdir, subdir)) if + os.path.isfile(os.path.join(scxdir, subdir, fn))] # ADM split off the extensions. exts = [os.path.splitext(fn)[1] for fn in fnswext] # ADM check they're all allowed extensions. @@ -284,8 +295,20 @@ def read_files(scxdir, scnd_mask): log.debug(' path: {}'.format(fn)) # ADM if the relevant file is a .txt file, read it in. if os.path.exists(fn+'.txt'): - scxin = np.loadtxt(fn+'.txt', usecols=[0, 1, 2, 3, 4, 5], - dtype=indatamodel.dtype) + try: + scxin = np.loadtxt(fn+'.txt', usecols=[0, 1, 2, 3, 4, 5], + dtype=indatamodel.dtype) + except (ValueError, IndexError): + msg = "First 6 columns don't correspond to {} in {}.txt".format( + indatamodel.dtype, fn) + # ADM perhaps people provided .csv files as .txt files. + try: + scxin = np.loadtxt(fn+'.txt', usecols=[0, 1, 2, 3, 4, 5], + dtype=indatamodel.dtype, delimiter=",") + except (ValueError, IndexError): + log.error(msg) + raise IOError(msg) + # ADM otherwise it's a fits file, read it in. else: scxin = fitsio.read(fn+'.fits', @@ -299,6 +322,15 @@ def read_files(scxdir, scnd_mask): for col in indatamodel.dtype.names: assert scxin[col].dtype == indatamodel[col].dtype, msg + # ADM check RA/Dec are reasonable. + outofbounds = ((scxin["RA"] >= 360.) | (scxin["RA"] < 0) | + (scxin["DEC"] > 90) | (scxin["DEC"] < -90)) + if np.any(outofbounds): + msg = "RA/Dec outside of range in {}; RA={}, Dec={}".format( + fn, scxin["RA"][outofbounds], scxin["DEC"][outofbounds]) + log.error(msg) + raise IOError(msg) + # ADM the default is 2015.5 for the REF_EPOCH. ii = scxin["REF_EPOCH"] == 0 scxin["REF_EPOCH"][ii] = 2015.5 @@ -316,6 +348,7 @@ def read_files(scxdir, scnd_mask): scxout["TARGETID"] = -1 scxout["OBSCONDITIONS"] = \ obsconditions.mask(scnd_mask[name].obsconditions) + scxout["PRIM_MATCH"] = False scxall.append(scxout) return np.concatenate(scxall) @@ -349,33 +382,58 @@ def add_primary_info(scxtargs, priminfodir): log.warning("No secondary target matches a primary target!!!") return scxtargs - primtargs = fitsio.read(primfns[0]) - for i in range(1, len(primfns)): - prim = fitsio.read(primfns[i]) - primtargs = np.concatenate([primtargs, prim]) + log.info("Begin reading files from {}...t={:.1f}s".format( + priminfodir, time()-start)) + primtargs = [] + for primfn in primfns: + prim = fitsio.read(primfn) + primtargs.append(prim) + primtargs = np.concatenate(primtargs) + log.info("Done reading files...t={:.1f}s".format(time()-start)) # ADM make a unique look-up for the target sets. scxbitnum = np.log2(scxtargs["SCND_TARGET"]).astype('int') primbitnum = np.log2(primtargs["SCND_TARGET"]).astype('int') - - scxids = 1000 * scxtargs["SCND_ORDER"] + scxbitnum - primids = 1000 * primtargs["SCND_ORDER"] + primbitnum - - # ADM if a secondary matched TWO (or more) primaries, - # ADM only retain the highest-priority primary. + # ADM SCND_ORDER can potentially run into the tens-of-millions, so + # ADM we need to use int64 type to get as high as 1000 x SCND_ORDER. + scxids = 1000 * scxtargs["SCND_ORDER"].astype('int64') + scxbitnum + primids = 1000 * primtargs["SCND_ORDER"].astype('int64') + primbitnum + + # ADM a primary (with the same TARGETID) could match TWO (or more) + # ADM secondaries. Resolve these on which matched a primary target + # ADM (rather than just a source from the sweeps) and then ALSO on + # ADM which has highest priority (via True/False * PRIORITY_INIT). alldups = [] - # for _, dups in duplicates(primids): for _, dups in duplicates(primtargs['TARGETID']): - am = np.argmax(primtargs[dups]["PRIORITY_INIT"]) + am = np.argmax(primtargs[dups]["PRIM_MATCH"]*primtargs[dups]["PRIORITY_INIT"]) dups = np.delete(dups, am) alldups.append(dups) - # ADM to catch the case of no duplicates when h-stacking. - if len(alldups) == 0: - alldups = [alldups] - alldups = np.hstack(alldups) - log.debug("Discarding {} primary duplicates".format(len(alldups))) - primtargs = np.delete(primtargs, alldups) - primids = np.delete(primids, alldups) + # ADM catch cases where there are no duplicates. + if len(alldups) != 0: + alldups = np.hstack(alldups) + primtargs = np.delete(primtargs, alldups) + primids = np.delete(primids, alldups) + log.info("Discard {} cases where a primary matched multiple secondaries". + format(len(alldups))) + + # ADM matches could also have occurred ACROSS HEALPixels, producing + # ADM duplicated secondary targets with different TARGETIDs... + alldups = [] + for _, dups in duplicates(primids): + # ADM...resolve these on which matched a primary target (rather + # ADM than just a source from a sweeps files) and then ALSO on + # ADM which has the highest priority. The combination in the code + # ADM is, e.g., True/False (meaning 1/0) * PRIORITY_INIT. + am = np.argmax(primtargs[dups]["PRIM_MATCH"]*primtargs[dups]["PRIORITY_INIT"]) + dups = np.delete(dups, am) + alldups.append(dups) + # ADM catch cases where there are no duplicates. + if len(alldups) != 0: + alldups = np.hstack(alldups) + primtargs = np.delete(primtargs, alldups) + primids = np.delete(primids, alldups) + log.info("Remove {} other cases where a secondary matched several primaries". + format(len(alldups))) # ADM we already know that all primaries match a secondary, so, # ADM for speed, we can reduce to the matching set. @@ -394,8 +452,9 @@ def add_primary_info(scxtargs, priminfodir): # ADM now we have the matches, update the secondary targets # ADM with the primary TARGETIDs. scxtargs["TARGETID"][scxii] = primtargs["TARGETID"][primii] + scxtargs["PRIM_MATCH"][scxii] = primtargs["PRIM_MATCH"][primii] - # APC Secondary targets that don't match to a primary. + # APC Secondary targets that don't match to a primary target. # APC all still have TARGETID = -1 at this point. They # APC get removed in finalize_secondary(). log.info("Done matching primaries in {} to secondaries...t={:.1f}s" @@ -405,7 +464,7 @@ def add_primary_info(scxtargs, priminfodir): def match_secondary(primtargs, scxdir, scndout, sep=1., - pix=None, nside=None): + pix=None, nside=None, swfiles=None): """Match secondary targets to primary targets and update bits. Parameters @@ -425,6 +484,11 @@ def match_secondary(primtargs, scxdir, scndout, sep=1., pix at the supplied `nside`, as a speed-up. nside : :class:`int`, optional, defaults to `None` The (NESTED) HEALPixel nside to be used with `pixlist`. + swfiles : :class:`list`, optional, defaults to `None` + A list of files (typically sweep files). If passed and not `None` + then once all of the primary TARGETS have been matched and the + relevant bit information updated, use these files to find + additional sources from which to derive a primary TARGETID. Returns ------- @@ -467,7 +531,7 @@ def match_secondary(primtargs, scxdir, scndout, sep=1., raise ValueError(msg) # ADM warn the user if the secondary and primary samples are "large". - big = 500000 + big = 1e6 if np.sum(inhp) > big and len(primtargs) > big: log.warning('Large secondary (N={}) and primary (N={}) samples' .format(np.sum(inhp), len(primtargs))) @@ -515,10 +579,14 @@ def match_secondary(primtargs, scxdir, scndout, sep=1., if np.any(scnd_update): # APC Allow changes to primaries if the DESI_TARGET bitmask has # APC only the following bits set, in any combination. - log.info('Testing if secondary targets can update {} matched primaries'.format(scnd_update.sum())) - update_from_scnd_bits = desi_mask['SCND_ANY'] | desi_mask['MWS_ANY'] | desi_mask['STD_BRIGHT'] | desi_mask['STD_FAINT'] | desi_mask['STD_WD'] + log.info('Test if secondaries can update {} matched primaries'.format( + scnd_update.sum())) + update_from_scnd_bits = (desi_mask['SCND_ANY'] | desi_mask['MWS_ANY'] | + desi_mask['STD_BRIGHT'] | desi_mask['STD_FAINT'] + | desi_mask['STD_WD']) scnd_update &= ((targs[desicols[0]] & ~update_from_scnd_bits) == 0) - log.info('Setting new priority, numobs and obsconditions from secondary for {} matched primaries'.format(scnd_update.sum())) + log.info('New priority, numobs, obscon for {} matched primaries'.format( + scnd_update.sum())) # APC Primary and secondary obsconditions are or'd scnd_obscon = set_obsconditions(targs[scnd_update], scnd=True) @@ -535,9 +603,12 @@ def match_secondary(primtargs, scxdir, scndout, sep=1., # APC secondaries can increase priority and numobs for edr, oc in zip(ender, obscon): pc, nc = "PRIORITY_INIT"+edr, "NUMOBS_INIT"+edr - scnd_priority, scnd_numobs = initial_priority_numobs(targs[scnd_update], obscon=oc, scnd=True) - targs[nc][scnd_update] = np.maximum(targs[nc][scnd_update], scnd_numobs) - targs[pc][scnd_update] = np.maximum(targs[pc][scnd_update], scnd_priority) + scnd_priority, scnd_numobs = initial_priority_numobs( + targs[scnd_update], obscon=oc, scnd=True) + targs[nc][scnd_update] = np.maximum( + targs[nc][scnd_update], scnd_numobs) + targs[pc][scnd_update] = np.maximum( + targs[pc][scnd_update], scnd_priority) # ADM update the secondary targets with the primary information. scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs] @@ -546,6 +617,58 @@ def match_secondary(primtargs, scxdir, scndout, sep=1., hipri = np.maximum(targs["PRIORITY_INIT_DARK"], targs["PRIORITY_INIT_BRIGHT"]) scxtargs["PRIORITY_INIT"][mscx] = hipri[mtargs] + # ADM record that we have a match to a primary. + scxtargs["PRIM_MATCH"][mscx] = True + + # ADM now we're done matching the primary and secondary targets, also + # ADM match the secondary targets to sweep files, if passes, to find + # ADM TARGETIDs. + notid = scxtargs["TARGETID"] == -1 + if swfiles is not None and np.sum(notid) > 0: + log.info('Reading input sweep files...t={:.1f}s'.format(time()-start)) + # ADM first read in all of the sweeps files. + swobjs = [] + for ifil, swfile in enumerate(swfiles): + swobj = fitsio.read(swfile, columns=["RELEASE", "BRICKID", "OBJID", + "RA", "DEC"]) + # ADM limit to just sources in the healpix of interest. + # ADM remembering to grab adjacent pixels for edge effects. + inhp = np.ones(len(swobj), dtype="?") + if nside is not None and pix is not None: + inhp = is_in_hp(swobj, nside, allpix) + swobjs.append(swobj[inhp]) + log.info("Read {} sources from {}/{} sweep files...t={:.1f}s".format( + np.sum(inhp), ifil+1, len(swfiles), time()-start)) + swobjs = np.concatenate(swobjs) + # ADM resolve so there are no duplicates across the N/S boundary. + swobjs = resolve(swobjs) + log.info("Total sources read: {}".format(len(swobjs))) + + # ADM continue if there are sources in the pixels of interest. + if len(swobjs) > 0: + # ADM limit to just secondaries in the healpix of interest. + inhp = np.ones(len(scxtargs), dtype="?") + if nside is not None and pix is not None: + inhp = is_in_hp(scxtargs, nside, pix) + + # ADM now perform the match. + log.info('Matching secondary targets to sweep files...t={:.1f}s' + .format(time()-start)) + mswobjs, mscx = radec_match_to(swobjs, + scxtargs[inhp & notid], sep=sep) + # ADM recast the indices to the full set of secondaries, + # ADM instead of just those that were in the relevant pixels. + mscx = np.where(inhp & notid)[0][mscx] + log.info('Found {} additional matches...t={:.1f}s'.format( + len(mscx), time()-start)) + + if len(mscx) > 0: + # ADM construct the targetid from the sweeps information. + targetid = encode_targetid(objid=swobjs['OBJID'], + brickid=swobjs['BRICKID'], + release=swobjs['RELEASE']) + # ADM and add the targetid to the secondary targets. + scxtargs["TARGETID"][mscx] = targetid[mswobjs] # ADM write the secondary targets that have updated TARGETIDs. ii = scxtargs["TARGETID"] != -1 @@ -667,7 +790,7 @@ def finalize_secondary(scxtargs, scnd_mask, survey='main', sep=1., # ADM check that the objid array was entirely populated. assert np.all(objid != -1) - # ADM assemble the TARGETID, SCND objects have RELEASE==0. + # ADM assemble the TARGETID, SCND objects. targetid = encode_targetid(objid=objid, brickid=brxid, release=release) # ADM a check that the generated TARGETIDs are unique. @@ -678,15 +801,16 @@ def finalize_secondary(scxtargs, scnd_mask, survey='main', sep=1., # ADM assign the unique TARGETIDs to the secondary objects. scxtargs["TARGETID"][nomatch] = targetid[nomatch] - log.debug("Assigned {} targetids to unmatched secondaries".format(len(targetid[nomatch]))) + log.debug("Assigned {} targetids to unmatched secondaries".format( + len(targetid[nomatch]))) # ADM match secondaries to themselves, to ensure duplicates # ADM share a TARGETID. Don't match special (OVERRIDE) targets # ADM or sources that have already been matched to a primary. w = np.where(~scxtargs["OVERRIDE"] & nomatch)[0] if len(w) > 0: - log.info("Matching secondary targets to themselves...t={:.1f}s" - .format(time()-t0)) + log.info("Matching {} secondary targets to themselves...t={:.1f}s" + .format(len(scxtargs), time()-t0)) # ADM use astropy for the matching. At NERSC, astropy matches # ADM ~20M objects to themselves in about 10 minutes. c = SkyCoord(scxtargs["RA"][w]*u.deg, scxtargs["DEC"][w]*u.deg) @@ -730,6 +854,9 @@ def finalize_secondary(scxtargs, scnd_mask, survey='main', sep=1., # on lowest index in list of duplicates dups = np.delete(dups, np.argmax(scxtargs['PRIORITY_INIT'][dups])) alldups.append(dups) + # ADM guard against the case that there are no duplicates. + if len(alldups) == 0: + alldups = [alldups] alldups = np.hstack(alldups) log.debug("Flagging {} duplicate secondary targetids with PRIORITY_INIT=-1".format(len(alldups))) @@ -764,7 +891,8 @@ def finalize_secondary(scxtargs, scnd_mask, survey='main', sep=1., done[pc], done[nc] = initial_priority_numobs(done, obscon=oc, scnd=True) # APC Flagged duplicates are removed in io.write_secondary - done[pc][alldups] = -1 + if len(alldups) > 0: + done[pc][alldups] = -1 # APC add secondary flag in DESI_TARGET cols, mx, surv = main_cmx_or_sv(done, scnd=True) diff --git a/py/desitarget/sv1/data/sv1_targetmask.yaml b/py/desitarget/sv1/data/sv1_targetmask.yaml index 8ba03046d..36d5b3cf2 100644 --- a/py/desitarget/sv1/data/sv1_targetmask.yaml +++ b/py/desitarget/sv1/data/sv1_targetmask.yaml @@ -136,6 +136,11 @@ sv1_mws_mask: - [MWS_MAIN_FAINT_NORTH, 15, "MWS magnitude limited sample from DECam", {obsconditions: BRIGHT|GRAY|DARK}] - [MWS_MAIN_FAINT_SOUTH, 16, "MWS magnitude limited sample from DECam", {obsconditions: BRIGHT|GRAY|DARK}] + # ADM Standard stars based only on Gaia. + - [GAIA_STD_FAINT, 33, "Standard stars for dark/gray conditions", {obsconditions: DARK|GRAY}] + - [GAIA_STD_WD, 34, "White Dwarf stars", {obsconditions: DARK|GRAY|BRIGHT}] + - [GAIA_STD_BRIGHT, 35, "Standard stars for BRIGHT conditions", {obsconditions: BRIGHT}] + # ADM back-up targets for poor conditions. - [BACKUP_BRIGHT, 60, "Bright backup Gaia targets", {obsconditions: DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18}] - [BACKUP_FAINT, 61, "Fainter backup Gaia targets", {obsconditions: DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18}] @@ -144,20 +149,54 @@ sv1_mws_mask: #- ADM SV secondary survey targets bit mask. #- ADM note that the 'filenames', here, should all be present in the #- ADM directory that corresponds to the $SECONDARY_DIR/sv1 environment -#- ADM variable, e.g. $SECONDARY_DIR/sv1/veto.fits for VETO targets. +#- ADM variable, e.g. $SECONDARY_DIR/sv1/veto.txt for VETO targets. sv1_scnd_mask: - - [VETO, 0, "Never observe, even if a primary target bit is set", + - [VETO, 0, "Never observe, even if a primary target bit is set", {obsconditions: DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18, filename: 'veto'}] - - [DR14Q, 1, "Known quasars from the SDSS DR14Q catalog", - {obsconditions: DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18, filename: 'dr14q'}] - - [MWS_CALIB, 10, "Stars with APOGEE/BOSS/GALAH/GAIAESO spectra", + - [UDG, 1, "See $SCND_DIR/UDG.txt", {obsconditions: DARK, filename: 'UDG'}] + - [FIRST_MALS, 2, "See $SCND_DIR/FIRST_MALS.txt", {obsconditions: DARK, filename: 'FIRST_MALS'}] + - [WD_BINARIES, 3, "See $SCND_DIR/WD_BINARIES.txt", {obsconditions: BRIGHT|DARK, filename: 'WD_BINARIES'}] + - [LBG_TOMOG, 4, "See $SCND_DIR/LBG_TOMOG.txt", {obsconditions: DARK, filename: 'LBG_TOMOG'}] + - [QSO_RED, 5, "See $SCND_DIR/QSO_RED.ipynb", {obsconditions: DARK, filename: 'QSO_RED'}] + - [M31_KNOWN, 6, "See $SCND_DIR/M31_KNOWN.txt", {obsconditions: DARK, filename: 'M31_KNOWN'}] + - [M31_QSO, 7, "See $SCND_DIR/M31_QSO.txt.", {obsconditions: DARK, filename: 'M31_QSO'}] + - [M31_STAR, 8, "See $SCND_DIR/M31_STAR.txt", {obsconditions: DARK, filename: 'M31_STAR'}] +# - [MWS_DDOGIANTS, 9, "See $SCND_DIR", {obsconditions: BRIGHT, filename: 'MWS_DDOGIANTS'}] + - [MWS_CLUS_GAL_DEEP, 10, "See $SCND_DIR/MWS_CLUS_GAL_DEEP.txt", {obsconditions: DARK, filename: 'MWS_CLUS_GAL_DEEP'}] + - [LOW_MASS_AGN, 11, "See $SCND_DIR/LOW_MASS_AGN.txt", {obsconditions: DARK, filename: 'LOW_MASS_AGN'}] + - [FAINT_HPM, 12, "See $SCND_DIR/FAINT_HPM.txt", {obsconditions: DARK, filename: 'FAINT_HPM'}] + - [GW190412, 13, "See $SCND_DIR/GW190412.ipynb", {obsconditions: BRIGHT|DARK, filename: 'GW190412'}] + - [IC134191, 14, "See $SCND_DIR/IC134191.ipynb", {obsconditions: BRIGHT|DARK, filename: 'IC134191'}] + - [PV_BRIGHT, 15, "See $SCND_DIR/PV_BRIGHT.ipynb", {obsconditions: BRIGHT, filename: 'PV_BRIGHT'}] + - [PV_DARK, 16, "See $SCND_DIR/PV_DARK.ipynb", {obsconditions: DARK, filename: 'PV_DARK'}] + - [LOW_Z, 17, "See $SCND_DIR/LOW_Z.ipynb", {obsconditions: DARK, filename: 'LOW_Z'}] + - [BHB, 18, "See $SCND_DIR/BHB.txt", {obsconditions: DARK, filename: 'BHB'}] + - [SPCV, 19, "See $SCND_DIR/SPCV.txt", {obsconditions: DARK, filename: 'SPCV'}] + - [DC3R2_GAMA, 20, "See $SCND_DIR/DC3R2_GAMA.ipynb", {obsconditions: DARK, filename: 'DC3R2_GAMA'}] + - [UNWISE_BLUE, 21, "See $SCND_DIR/UNWISE_BLUE.txt", {obsconditions: DARK, filename: 'UNWISE_BLUE'}] + - [UNWISE_GREEN, 22, "See $SCND_DIR/UNWISE_GREEN.txt", {obsconditions: DARK, filename: 'UNWISE_GREEN'}] + - [HETDEX_MAIN, 23, "See $SCND_DIR/HETDEX_MAIN.txt", {obsconditions: DARK, filename: 'HETDEX_MAIN'}] + - [HETDEX_HP, 24, "See $SCND_DIR/HETDEX_HP.txt", {obsconditions: DARK, filename: 'HETDEX_HP'}] +# - [PSF_OUT_BRIGHT, 25, "See $SCND_DIR", {obsconditions: BRIGHT, filename: 'PSF_OUT_BRIGHT'}] +# - [PSF_OUT_DARK, 26, "See $SCND_DIR", {obsconditions: DARK, filename: 'PSF_OUT_DARK'}] + - [HPM_SOUM, 27, "See $SCND_DIR/HPM_SOUM.txt", {obsconditions: DARK, filename: 'HPM_SOUM'}] + - [SN_HOSTS, 28, "See $SCND_DIR/SN_HOSTS.txt", {obsconditions: DARK, filename: 'SN_HOSTS'}] + - [GAL_CLUS_BCG, 29, "See $SCND_DIR/GAL_CLUS_BCG.txt", {obsconditions: DARK, filename: 'GAL_CLUS_BCG'}] + - [GAL_CLUS_2ND, 30, "See $SCND_DIR/GAL_CLUS_2ND.txt", {obsconditions: DARK, filename: 'GAL_CLUS_2ND'}] + - [GAL_CLUS_SAT, 31, "See $SCND_DIR/GAL_CLUS_SAT.txt", {obsconditions: DARK, filename: 'GAL_CLUS_SAT'}] + - [HSC_HIZ_SNE, 32, "See $SCND_DIR/HSC_HIZ_SNE.txt", {obsconditions: DARK, filename: 'HSC_HIZ_SNE'}] + - [ISM_CGM_QGP, 33, "See $SCND_DIR/ISM_CGM_QGP.txt", {obsconditions: DARK, filename: 'ISM_CGM_QGP'}] + - [STRONG_LENS, 34, "See $SCND_DIR/STRONG_LENS.txt", {obsconditions: DARK, filename: 'STRONG_LENS'}] + - [WISE_VAR_QSO, 35, "See $SCND_DIR/WISE_VAR_QSO.txt", {obsconditions: DARK, filename: 'WISE_VAR_QSO'}] + - [MWS_CALIB, 36, "Stars with APOGEE/BOSS/GALAH/GAIAESO spectra", {obsconditions: DARK|GRAY|BRIGHT, filename: 'MWS_CALIB'}] - - [BACKUP_CALIB, 11, "Very bright stars with APOGEE/BOSS/GALAH/GAIAESO spectra", + - [BACKUP_CALIB, 37, "Very bright stars with APOGEE/BOSS/GALAH/GAIAESO spectra", {obsconditions: POOR|TWILIGHT12|TWILIGHT18, filename: 'BACKUP_CALIB'}] - - [MWS_MAIN_CLUSTER_SV, 13, "Main survey low mass and metal poor stars (OC/GC/dwarf/stream members)", + - [MWS_MAIN_CLUSTER_SV, 38, "Main survey low mass and metal poor stars (OC/GC/dwarf/stream members)", {obsconditions: DARK|GRAY|BRIGHT, filename: 'MWS_MAIN_CLUSTER_SV'}] - - [MWS_MAIN_RRLYR, 14, "Main survey halo tracers (RR Lyrae)", - {obsconditions: DARK|GRAY|BRIGHT, filename: 'MWS_MAIN_RRLYR'}] + - [MWS_RRLYR, 39, "Main survey halo tracers (RR Lyrae)", + {obsconditions: DARK|GRAY|BRIGHT, filename: 'MWS_RRLYR'}] + - [BRIGHT_HPM, 40, "See $SCND_DIR/BRIGHT_HPM.txt", {obsconditions: BRIGHT, filename: 'BRIGHT_HPM'}] #- Observation State #- if a target passes more than one target bit, it is possible that one bit @@ -301,15 +340,54 @@ priorities: BACKUP_BRIGHT: {UNOBS: 9, DONE: 9, OBS: 9, DONOTOBSERVE: 0} BACKUP_FAINT: {UNOBS: 8, DONE: 8, OBS: 8, DONOTOBSERVE: 0} BACKUP_VERY_FAINT: {UNOBS: 7, DONE: 7, OBS: 7, DONOTOBSERVE: 0} + # ADM Standards are special; priorities don't apply. + GAIA_STD_FAINT: -1 + GAIA_STD_WD: -1 + GAIA_STD_BRIGHT: -1 # ADM secondary target priorities. Probably all have very low UNOBS sv1_scnd_mask: - VETO: {UNOBS: 0, DONE: 0, OBS: 0, DONOTOBSERVE: 0} - DR14Q: {UNOBS: 10, DONE: 2, OBS: 1, DONOTOBSERVE: 0} - MWS_CALIB: {UNOBS: 2995, DONE: 2, OBS: 1, DONOTOBSERVE: 0} - BACKUP_CALIB: SAME_AS_MWS_CALIB - MWS_MAIN_CLUSTER_SV: {UNOBS: 1450, DONE: 400, OBS: 1, DONOTOBSERVE: 0} - MWS_MAIN_RRLYR: {UNOBS: 1450, DONE: 400, OBS: 1, DONOTOBSERVE: 0} + VETO: {UNOBS: 0, DONE: 0, OBS: 0, DONOTOBSERVE: 0} + UDG: {UNOBS: 1900, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + FIRST_MALS: {UNOBS: 1025, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + WD_BINARIES: {UNOBS: 1998, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + LBG_TOMOG: {UNOBS: 1800, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + QSO_RED: {UNOBS: 3400, MORE_ZGOOD: 3350, MORE_ZWARN: 3300, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + M31_KNOWN: SAME_AS_LBG_TOMOG + M31_QSO: SAME_AS_LBG_TOMOG + M31_STAR: SAME_AS_LBG_TOMOG +# MWS_DDOGIANTS: {UNOBS: 1450, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + MWS_CLUS_GAL_DEEP: {UNOBS: 4000, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + LOW_MASS_AGN: {UNOBS: 1025, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + FAINT_HPM: SAME_AS_LOW_MASS_AGN + GW190412: {UNOBS: 5000, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + IC134191: SAME_AS_GW190412 + PV_BRIGHT: {UNOBS: 100, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + PV_DARK: {UNOBS: 100, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + LOW_Z: {UNOBS: 90, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + BHB: {UNOBS: 1950, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + SPCV: SAME_AS_LOW_MASS_AGN + DC3R2_GAMA: SAME_AS_LOW_MASS_AGN + UNWISE_BLUE: SAME_AS_LBG_TOMOG + UNWISE_GREEN: SAME_AS_LBG_TOMOG + HETDEX_MAIN: SAME_AS_LBG_TOMOG + HETDEX_HP: SAME_AS_LBG_TOMOG +# PSF_OUT_BRIGHT: {UNOBS: 70, DONE: 2, OBS: 1, DONOTOBSERVE: 0} +# PSF_OUT_DARK: SAME_AS_PSF_OUT_BRIGHT + HPM_SOUM: SAME_AS_LOW_MASS_AGN + SN_HOSTS: SAME_AS_BHB + GAL_CLUS_BCG: {UNOBS: 1025, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + GAL_CLUS_2ND: {UNOBS: 1024, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + GAL_CLUS_SAT: {UNOBS: 200, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + HSC_HIZ_SNE: SAME_AS_LBG_TOMOG + ISM_CGM_QGP: {UNOBS: 4000, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + STRONG_LENS: SAME_AS_UDG + WISE_VAR_QSO: SAME_AS_QSO_RED + MWS_CALIB: {UNOBS: 2995, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + BACKUP_CALIB: SAME_AS_MWS_CALIB + MWS_MAIN_CLUSTER_SV: {UNOBS: 1450, DONE: 400, OBS: 1, DONOTOBSERVE: 0} + MWS_RRLYR: {UNOBS: 1450, DONE: 400, OBS: 1, DONOTOBSERVE: 0} + BRIGHT_HPM: SAME_AS_LOW_MASS_AGN # ADM INITIAL number of observations (NUMOBS) for each target bit # ADM SAME_AS_XXX means to use the NUMOBS for bitname XXX @@ -424,12 +502,51 @@ numobs: BACKUP_BRIGHT: 1 BACKUP_FAINT: SAME_AS_BACKUP_BRIGHT BACKUP_VERY_FAINT: SAME_AS_BACKUP_BRIGHT + # ADM Standards are special; numobs doesn't apply. + GAIA_STD_FAINT: -1 + GAIA_STD_WD: -1 + GAIA_STD_BRIGHT: -1 # ADM initial number of observations for secondary targets sv1_scnd_mask: - VETO: 1 - DR14Q: 1 - MWS_CALIB: 100 - MWS_MAIN_CLUSTER_SV: 1 - MWS_MAIN_RRLYR: 100 - BACKUP_CALIB: SAME_AS_MWS_CALIB + VETO: 1 + UDG: 100 + FIRST_MALS: 100 + WD_BINARIES: 100 + LBG_TOMOG: 100 + QSO_RED: 100 + M31_KNOWN: 100 + M31_QSO: 100 + M31_STAR: 100 +# MWS_DDOGIANTS: 100 + MWS_CLUS_GAL_DEEP: 100 + LOW_MASS_AGN: 100 + FAINT_HPM: 100 + GW190412: 100 + IC134191: 100 + PV_BRIGHT: 100 + PV_DARK: 100 + LOW_Z: 100 + BHB: 100 + SPCV: 100 + DC3R2_GAMA: 100 + UNWISE_BLUE: 100 + UNWISE_GREEN: 100 + HETDEX_MAIN: 100 + HETDEX_HP: 100 +# PSF_OUT_BRIGHT: 100 +# PSF_OUT_DARK: 100 + HPM_SOUM: 100 + SN_HOSTS: 100 + GAL_CLUS_BCG: 100 + GAL_CLUS_2ND: 100 + GAL_CLUS_SAT: 100 + HSC_HIZ_SNE: 100 + ISM_CGM_QGP: 100 + STRONG_LENS: 100 + WISE_VAR_QSO: 100 + MWS_CALIB: 100 + MWS_MAIN_CLUSTER_SV: 1 + MWS_RRLYR: 100 + BACKUP_CALIB: SAME_AS_MWS_CALIB + BRIGHT_HPM: 100 \ No newline at end of file diff --git a/py/desitarget/sv1/sv1_cuts.py b/py/desitarget/sv1/sv1_cuts.py index c3a622a1a..fd99feb3f 100644 --- a/py/desitarget/sv1/sv1_cuts.py +++ b/py/desitarget/sv1/sv1_cuts.py @@ -17,10 +17,15 @@ from time import time from pkg_resources import resource_filename +import healpy as hp +import fitsio + +import astropy.units as u +from astropy.coordinates import SkyCoord from desitarget.cuts import _getColors, _psflike, _check_BGS_targtype_sv from desitarget.cuts import shift_photo_north -from desitarget.gaiamatch import is_in_Galaxy +from desitarget.gaiamatch import is_in_Galaxy, find_gaia_files_hp from desitarget.geomask import imaging_mask # ADM set up the DESI default logger @@ -31,6 +36,129 @@ start = time() +def isGAIA_STD(ra=None, dec=None, galb=None, gaiaaen=None, pmra=None, pmdec=None, + parallax=None, parallaxovererror=None, gaiabprpfactor=None, + gaiasigma5dmax=None, gaiagmag=None, gaiabmag=None, gaiarmag=None, + gaiadupsource=None, gaiaparamssolved=None, + primary=None, test=False, nside=2): + """Standards based solely on Gaia data. + + Parameters + ---------- + test : :class:`bool`, optional, defaults to ``False`` + If ``True``, then we're running unit tests and don't have to + find and read every possible Gaia file. + nside : :class:`int`, optional, defaults to 2 + (NESTED) HEALPix nside, if targets are being parallelized. + The default of 2 should be benign for serial processing. + + Returns + ------- + :class:`array_like` + ``True`` if the object is a bright "GAIA_STD_FAINT" target. + :class:`array_like` + ``True`` if the object is a faint "GAIA_STD_BRIGHT" target. + :class:`array_like` + ``True`` if the object is a white dwarf "GAIA_STD_WD" target. + + Notes + ----- + - See :func:`~desitarget.cuts.set_target_bits` for other parameters. + - Current version (01/08/21) is version XXX on `the SV wiki`_. + """ + if primary is None: + primary = np.ones_like(gaiagmag, dtype='?') + + # ADM restrict all classes to dec >= -30. + primary &= dec >= -30. + std = primary.copy() + + # ADM the regular "standards" codes need to know whether something has + # ADM a Gaia match. Here, everything is a Gaia match. + gaia = np.ones_like(gaiagmag, dtype='?') + + # ADM determine the Gaia-based white dwarf standards. + std_wd = isMWS_WD( + primary=primary, gaia=gaia, galb=galb, astrometricexcessnoise=gaiaaen, + pmra=pmra, pmdec=pmdec, parallax=parallax, + parallaxovererror=parallaxovererror, photbprpexcessfactor=gaiabprpfactor, + astrometricsigma5dmax=gaiasigma5dmax, gaiagmag=gaiagmag, + gaiabmag=gaiabmag, gaiarmag=gaiarmag + ) + + # ADM apply the Gaia quality cuts for standards. + std &= isSTD_gaia(primary=primary, gaia=gaia, astrometricexcessnoise=gaiaaen, + pmra=pmra, pmdec=pmdec, parallax=parallax, + dupsource=gaiadupsource, paramssolved=gaiaparamssolved, + gaiagmag=gaiagmag, gaiabmag=gaiabmag, gaiarmag=gaiarmag) + + # ADM restrict to point sources. + ispsf = np.logical_or( + (gaiagmag <= 19.) * (gaiaaen < 10.**0.5), + (gaiagmag >= 19.) * (gaiaaen < 10.**(0.5 + 0.2*(gaiagmag - 19.))) + ) + std &= ispsf + + # ADM apply the Gaia color cuts for standards. + bprp = gaiabmag - gaiarmag + gbp = gaiagmag - gaiabmag + std &= bprp > 0.2 + std &= bprp < 0.9 + std &= gbp > -1.*bprp/2.0 + std &= gbp < 0.3-bprp/2.0 + + # ADM remove any sources that have neighbors in Gaia within 3.5"... + # ADM for speed, run only sources for which std is still True. + log.info("Isolating Gaia-only standards...t={:.1f}s".format(time()-start)) + ii_true = np.where(std)[0] + if len(ii_true) > 0: + # ADM determine the pixels of interest. + theta, phi = np.radians(90-dec), np.radians(ra) + pixlist = list(set(hp.ang2pix(nside, theta, phi, nest=True))) + # ADM read in the necessary Gaia files. + fns = find_gaia_files_hp(nside, pixlist, neighbors=True) + gaiaobjs = [] + gaiacols = ["RA", "DEC", "PHOT_G_MEAN_MAG", "PHOT_RP_MEAN_MAG"] + for i, fn in enumerate(fns): + if i % 25 == 0: + log.info("Read {}/{} files for Gaia-only standards...t={:.1f}s" + .format(i, len(fns), time()-start)) + try: + gaiaobjs.append(fitsio.read(fn, columns=gaiacols)) + except OSError: + if test: + pass + else: + msg = "failed to find or open the following file: (ffopen) " + msg += fn + log.critical(msg) + raise OSError + gaiaobjs = np.concatenate(gaiaobjs) + # ADM match the standards to the broader Gaia sources at 3.5". + matchrad = 3.5*u.arcsec + cstd = SkyCoord(ra[ii_true]*u.degree, dec[ii_true]*u.degree) + cgaia = SkyCoord(gaiaobjs["RA"]*u.degree, gaiaobjs["DEC"]*u.degree) + idstd, idgaia, d2d, _ = cgaia.search_around_sky(cstd, matchrad) + # ADM remove source matches with d2d=0 (i.e. the source itself!). + idgaia, idstd = idgaia[d2d > 0], idstd[d2d > 0] + # ADM remove matches within 5 mags of a Gaia source. + badmag = ( + (gaiagmag[ii_true][idstd] + 5 > gaiaobjs["PHOT_G_MEAN_MAG"][idgaia]) | + (gaiarmag[ii_true][idstd] + 5 > gaiaobjs["PHOT_RP_MEAN_MAG"][idgaia])) + std[ii_true[idstd][badmag]] = False + + # ADM add the brightness cuts in Gaia G-band. + std_bright = std.copy() + std_bright &= gaiagmag >= 15 + std_bright &= gaiagmag < 18 + + std_faint = std.copy() + std_faint &= gaiagmag >= 16 + std_faint &= gaiagmag < 19 + + return std_faint, std_bright, std_wd + + def isBACKUP(ra=None, dec=None, gaiagmag=None, primary=None): """BACKUP targets based on Gaia magnitudes. @@ -382,7 +510,7 @@ def isSTD_gaia(primary=None, gaia=None, astrometricexcessnoise=None, see :func:`~desitarget.sv1.sv1_cuts.isSTD` for other details. """ if primary is None: - primary = np.ones_like(gflux, dtype='?') + primary = np.ones_like(gaiagmag, dtype='?') std = primary.copy() # ADM Bp and Rp are both measured. @@ -1454,7 +1582,7 @@ def isMWS_WD(primary=None, gaia=None, galb=None, astrometricexcessnoise=None, Parameters ---------- - see :func:`~desitarget.sv1.sv1_cuts.set_target_bits` for other parameters. + see :func:`~desitarget.sv1.sv1_cuts.set_target_bits` for parameters. Returns ------- diff --git a/py/desitarget/targets.py b/py/desitarget/targets.py index 6d05a6dd7..2adf59310 100644 --- a/py/desitarget/targets.py +++ b/py/desitarget/targets.py @@ -607,7 +607,7 @@ def calc_priority(targets, zcat, obscon, state=False): # DESI dark time targets. if survey != 'cmx': if desi_target in targets.dtype.names: - # ADM set initialstate of CALIB for potential calibration targets. + # ADM set initial state of CALIB for potential calibration targets. names = ('SKY', 'BAD_SKY', 'SUPP_SKY', 'STD_FAINT', 'STD_WD', 'STD_BRIGHT') for name in names: @@ -686,23 +686,29 @@ def calc_priority(targets, zcat, obscon, state=False): # MWS targets. if mws_target in targets.dtype.names: + # ADM set initial state of CALIB for potential calibration targets. + stdnames = ('GAIA_STD_FAINT', 'GAIA_STD_WD', 'GAIA_STD_BRIGHT') for name in mws_mask.names(): # ADM only update priorities for passed observing conditions. pricon = obsconditions.mask(mws_mask[name].obsconditions) if (obsconditions.mask(obscon) & pricon) != 0: ii = (targets[mws_target] & mws_mask[name]) != 0 - for sbool, sname in zip( - [unobs, done, zgood, zwarn], - ["UNOBS", "DONE", "MORE_ZGOOD", "MORE_ZWARN"] - ): - # ADM update priorities and target states. - Mxp = mws_mask[name].priorities[sname] - # ADM update states BEFORE changing priorities. - ts = "{}|{}".format("MWS", sname) - target_state[ii & sbool] = np.where( - priority[ii & sbool] < Mxp, ts, target_state[ii & sbool]) - priority[ii & sbool] = np.where( - priority[ii & sbool] < Mxp, Mxp, priority[ii & sbool]) + # ADM standards have no priority. + if name in stdnames: + target_state[ii] = "CALIB" + else: + for sbool, sname in zip( + [unobs, done, zgood, zwarn], + ["UNOBS", "DONE", "MORE_ZGOOD", "MORE_ZWARN"] + ): + # ADM update priorities and target states. + Mxp = mws_mask[name].priorities[sname] + # ADM update states BEFORE changing priorities. + ts = "{}|{}".format("MWS", sname) + target_state[ii & sbool] = np.where( + priority[ii & sbool] < Mxp, ts, target_state[ii & sbool]) + priority[ii & sbool] = np.where( + priority[ii & sbool] < Mxp, Mxp, priority[ii & sbool]) # ADM Secondary targets. if scnd_target in targets.dtype.names: @@ -819,7 +825,7 @@ def resolve(targets): ---------- targets : :class:`~numpy.ndarray` Rec array of targets. Must have columns "RA" and "DEC" and - either "RELEASE" or "PHOTSYS". + either "RELEASE" or "PHOTSYS" or "TARGETID". Returns ------- @@ -833,7 +839,11 @@ def resolve(targets): if 'PHOTSYS' in targets.dtype.names: photsys = targets["PHOTSYS"] else: - photsys = release_to_photsys(targets["RELEASE"]) + if 'RELEASE' in targets.dtype.names: + photsys = release_to_photsys(targets["RELEASE"]) + else: + _, _, release, _, _, _ = decode_targetid(targets["TARGETID"]) + photsys = release_to_photsys(release) # ADM a flag of which targets are from the 'N' photometry. from desitarget.cuts import _isonnorthphotsys diff --git a/py/desitarget/test/test_cuts.py b/py/desitarget/test/test_cuts.py index 6581c9a98..93d7eba2b 100644 --- a/py/desitarget/test/test_cuts.py +++ b/py/desitarget/test/test_cuts.py @@ -362,8 +362,9 @@ def test_backup(self): # ADM BACKUP targets can only run on the sweep files. for filelist in self.sweepfiles: # ADM limit to pixels covered in the Gaia unit test files. - targets = cuts.select_targets(filelist, numproc=1, tcnames=tc, - nside=self.nside, pixlist=self.pix) + targets = cuts.select_targets( + filelist, numproc=1, tcnames=tc, test=True, nside=self.nside, + pixlist=self.pix) self.assertTrue('DESI_TARGET' in targets.dtype.names) self.assertTrue('BGS_TARGET' in targets.dtype.names) self.assertTrue('MWS_TARGET' in targets.dtype.names) diff --git a/py/desitarget/test/test_priorities.py b/py/desitarget/test/test_priorities.py index 5c84424fc..d79d6496e 100644 --- a/py/desitarget/test/test_priorities.py +++ b/py/desitarget/test/test_priorities.py @@ -132,8 +132,9 @@ def test_priorities(self): # ADM should never trump the other bits. bgs_names = [name for name in bgs_mask.names() if 'NORTH' not in name and 'SOUTH' not in name] - mws_names = [name for name in mws_mask.names() if 'NORTH' not in name - and 'SOUTH' not in name and 'BACKUP' not in name] + mws_names = [name for name in mws_mask.names() if + 'NORTH' not in name and 'SOUTH' not in name and + 'BACKUP' not in name and 'STD' not in name] lowest_mws_priority_unobs = [mws_mask[n].priorities['UNOBS'] for n in mws_names] @@ -172,7 +173,7 @@ def test_mask_priorities(self): for mask in [desi_mask, bgs_mask, mws_mask]: for name in mask.names(): if ( - name.startswith('STD') or name.endswith('BRIGHT_OBJECT') or name in + 'STD' in name or name.endswith('BRIGHT_OBJECT') or name in ['BGS_ANY', 'MWS_ANY', 'SCND_ANY', 'SKY', 'SUPP_SKY', 'NO_TARGET'] ): self.assertEqual(mask[name].priorities, {}, 'mask.{} has priorities?'.format(name)) diff --git a/py/desitarget/test/test_sv.py b/py/desitarget/test/test_sv.py index ebd5d2211..70980c986 100644 --- a/py/desitarget/test/test_sv.py +++ b/py/desitarget/test/test_sv.py @@ -56,7 +56,7 @@ def test_sv_cuts(self): # ADM set backup to False as the Gaia unit test # ADM files only cover a limited pixel range. targets = cuts.select_targets(filelist, survey=survey, - backup=False) + backup=False, test=True) for col in desicol, bgscol, mwscol: self.assertTrue(col in targets.dtype.names) self.assertEqual(len(targets), np.count_nonzero(targets[desicol])) @@ -71,7 +71,7 @@ def test_sv_cuts(self): for filelist in self.sweepfiles: # ADM also test the backup targets in the pixels covered # ADM by the Gaia unit test files. - targets = cuts.select_targets(filelist, survey=survey, + targets = cuts.select_targets(filelist, survey=survey, test=True, nside=self.nside, pixlist=self.pix) for col in desicol, bgscol, mwscol: self.assertTrue(col in targets.dtype.names) diff --git a/py/desitarget/test/test_units.py b/py/desitarget/test/test_units.py index 208224b92..a8a7ab0b1 100644 --- a/py/desitarget/test/test_units.py +++ b/py/desitarget/test/test_units.py @@ -54,7 +54,7 @@ def test_fits_units(self): self.assertEqual(list(uniq), parsed) def test_quantities(self): - """Test all data model quantities in are in the units yaml file. + """Test all data model quantities are in the units yaml file. """ missing = [dmn for dmn in self.dmnames if dmn not in self.units] msg = 'These quantities are missing in {}'.format(self.fn)