From 422448fb64fc6eae4bb439db861411649de15376 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Wed, 30 Dec 2020 08:53:49 -0800 Subject: [PATCH 01/30] match to sweeps files to populate correct TARGETIDs even when a secondary target isn't a primary target --- bin/select_sv_targets | 4 +- bin/select_targets | 4 +- py/desitarget/io.py | 2 +- py/desitarget/secondary.py | 90 +++++++++++++++++++++++++++++++------- 4 files changed, 80 insertions(+), 20 deletions(-) diff --git a/bin/select_sv_targets b/bin/select_sv_targets index ee57153f9..349375e02 100755 --- a/bin/select_sv_targets +++ b/bin/select_sv_targets @@ -164,7 +164,7 @@ if ns.bundlefiles is None: scxdir = _get_scxdir(ns.scnddir) # ADM construct a label for the secondary file for TARGETID look-ups. # ADM use RELEASE to determine the release string for the targets. - drint = np.max(targets['RELEASE']//1000) + drint = int(np.max(targets['RELEASE']//1000)) scndoutdn = "priminfo-dr{}-{}".format(drint, desitarget_version) scndoutdn = os.path.join(scxdir, survey, "outdata", scndoutdn) if not os.path.exists(scndoutdn): @@ -181,7 +181,7 @@ if ns.bundlefiles is None: scndout = os.path.join(scndoutdn, scndoutfn) log.info("writing files of primary matches to...{}".format(scndout)) targets = match_secondary(targets, scxdir, scndout, sep=1., - pix=pixlist, nside=ns.nside) + pix=pixlist, nside=ns.nside, swfiles=infn) if ns.mask: targets = mask_targets(targets, inmaskfile=ns.mask, nside=nside) diff --git a/bin/select_targets b/bin/select_targets index 66c0ed7e8..b797d24d3 100755 --- a/bin/select_targets +++ b/bin/select_targets @@ -165,7 +165,7 @@ if ns.bundlefiles is None: scxdir = _get_scxdir(ns.scnddir) # ADM construct a label for the secondary file for TARGETID look-ups. # ADM use RELEASE to determine the release string for the targets. - drint = np.max(targets['RELEASE']//1000) + drint = int(np.max(targets['RELEASE']//1000)) scndoutdn = "priminfo-dr{}-{}".format(drint, desitarget_version) scndoutdn = os.path.join(scxdir, "outdata", scndoutdn) if not os.path.exists(scndoutdn): @@ -182,7 +182,7 @@ if ns.bundlefiles is None: scndout = os.path.join(scndoutdn, scndoutfn) log.info("writing files of primary matches to...{}".format(scndout)) targets = match_secondary(targets, scxdir, scndout, sep=1., - pix=pixlist, nside=ns.nside) + pix=pixlist, nside=ns.nside, swfiles=infn) if ns.mask: targets = mask_targets(targets, inmaskfile=ns.mask, nside=nside) diff --git a/py/desitarget/io.py b/py/desitarget/io.py index d01f0f5fa..ac912ff72 100644 --- a/py/desitarget/io.py +++ b/py/desitarget/io.py @@ -864,7 +864,7 @@ def write_secondary(targdir, data, primhdr=None, scxdir=None, obscon=None, header of the output `filename`. obscon : :class:`str`, optional, defaults to `None` Can pass one of "DARK" or "BRIGHT". If passed, don't write the - full set of secondary target that do not match a primary, + full set of secondary targets that do not match a primary, rather only write targets appropriate for "DARK|GRAY" or "BRIGHT" observing conditions. The relevant `PRIORITY_INIT` and `NUMOBS_INIT` columns will be derived from diff --git a/py/desitarget/secondary.py b/py/desitarget/secondary.py index d6ee164c4..ce91e76c3 100644 --- a/py/desitarget/secondary.py +++ b/py/desitarget/secondary.py @@ -369,13 +369,12 @@ def add_primary_info(scxtargs, priminfodir): am = np.argmax(primtargs[dups]["PRIORITY_INIT"]) dups = np.delete(dups, am) alldups.append(dups) - # ADM to catch the case of no duplicates when h-stacking. - if len(alldups) == 0: - alldups = [alldups] - alldups = np.hstack(alldups) - log.debug("Discarding {} primary duplicates".format(len(alldups))) - primtargs = np.delete(primtargs, alldups) - primids = np.delete(primids, alldups) + # ADM catch cases where there are no duplicates. + if len(alldups) != 0: + alldups = np.hstack(alldups) + primtargs = np.delete(primtargs, alldups) + primids = np.delete(primids, alldups) + log.debug("Discarded {} primary duplicates".format(len(alldups))) # ADM we already know that all primaries match a secondary, so, # ADM for speed, we can reduce to the matching set. @@ -405,7 +404,7 @@ def add_primary_info(scxtargs, priminfodir): def match_secondary(primtargs, scxdir, scndout, sep=1., - pix=None, nside=None): + pix=None, nside=None, swfiles=None): """Match secondary targets to primary targets and update bits. Parameters @@ -425,6 +424,11 @@ def match_secondary(primtargs, scxdir, scndout, sep=1., pix at the supplied `nside`, as a speed-up. nside : :class:`int`, optional, defaults to `None` The (NESTED) HEALPixel nside to be used with `pixlist`. + swfiles : :class:`list`, optional, defaults to `None` + A list of files (typically sweep files). If passed and not `None` + then once all of the primary TARGETS have been matched and the + relevant bit information updated, use these files to find + additional sources from which to derive a primary TARGETID. Returns ------- @@ -515,10 +519,14 @@ def match_secondary(primtargs, scxdir, scndout, sep=1., if np.any(scnd_update): # APC Allow changes to primaries if the DESI_TARGET bitmask has # APC only the following bits set, in any combination. - log.info('Testing if secondary targets can update {} matched primaries'.format(scnd_update.sum())) - update_from_scnd_bits = desi_mask['SCND_ANY'] | desi_mask['MWS_ANY'] | desi_mask['STD_BRIGHT'] | desi_mask['STD_FAINT'] | desi_mask['STD_WD'] + log.info('Test if secondaries can update {} matched primaries'.format( + scnd_update.sum())) + update_from_scnd_bits = (desi_mask['SCND_ANY'] | desi_mask['MWS_ANY'] | + desi_mask['STD_BRIGHT'] | desi_mask['STD_FAINT'] + | desi_mask['STD_WD']) scnd_update &= ((targs[desicols[0]] & ~update_from_scnd_bits) == 0) - log.info('Setting new priority, numobs and obsconditions from secondary for {} matched primaries'.format(scnd_update.sum())) + log.info('New priority, numobs, obscon for {} matched primaries'.format( + scnd_update.sum())) # APC Primary and secondary obsconditions are or'd scnd_obscon = set_obsconditions(targs[scnd_update], scnd=True) @@ -535,9 +543,12 @@ def match_secondary(primtargs, scxdir, scndout, sep=1., # APC secondaries can increase priority and numobs for edr, oc in zip(ender, obscon): pc, nc = "PRIORITY_INIT"+edr, "NUMOBS_INIT"+edr - scnd_priority, scnd_numobs = initial_priority_numobs(targs[scnd_update], obscon=oc, scnd=True) - targs[nc][scnd_update] = np.maximum(targs[nc][scnd_update], scnd_numobs) - targs[pc][scnd_update] = np.maximum(targs[pc][scnd_update], scnd_priority) + scnd_priority, scnd_numobs = initial_priority_numobs( + targs[scnd_update], obscon=oc, scnd=True) + targs[nc][scnd_update] = np.maximum( + targs[nc][scnd_update], scnd_numobs) + targs[pc][scnd_update] = np.maximum( + targs[pc][scnd_update], scnd_priority) # ADM update the secondary targets with the primary information. scxtargs["TARGETID"][mscx] = targs["TARGETID"][mtargs] @@ -547,6 +558,51 @@ def match_secondary(primtargs, scxdir, scndout, sep=1., targs["PRIORITY_INIT_BRIGHT"]) scxtargs["PRIORITY_INIT"][mscx] = hipri[mtargs] + # ADM now we're done matching the primary and secondary targets, also + # ADM match the secondary targets to sweep files, if passes, to find + # ADM TARGETIDs. + notid = scxtargs["TARGETID"] == -1 + if swfiles is not None and np.sum(notid) > 0: + log.info('Reading input sweep files...t={:.1f}s'.format(time()-start)) + # ADM first read in all of the sweeps files. + swobjs = [] + for ifil, swfile in enumerate(swfiles): + swobj = fitsio.read(swfile, columns=["RELEASE", "BRICKID", "OBJID", + "RA", "DEC"]) + # ADM limit to just sources in the healpix of interest. + # ADM remembering to grab adjacent pixels for edge effects. + inhp = np.ones(len(swobj), dtype="?") + if nside is not None and pix is not None: + inhp = is_in_hp(swobj, nside, allpix) + swobjs.append(swobj[inhp]) + log.info("Read {} sources from {}/{} sweep files...t={:.1f}s".format( + np.sum(inhp), ifil+1, len(swfiles), time()-start)) + swobjs = np.concatenate(swobjs) + log.info("Total sources read: {}".format(len(swobjs))) + + # ADM limit to just secondary targets in the healpix of interest. + inhp = np.ones(len(scxtargs), dtype="?") + if nside is not None and pix is not None: + inhp = is_in_hp(scxtargs, nside, pix) + + # ADM now perform the match. + log.info('Matching secondary targets to input sweep files...t={:.1f}s' + .format(time()-start)) + mswobjs, mscx = radec_match_to(swobjs, scxtargs[inhp & notid], sep=sep) + # ADM recast the indices to the full set of secondary targets, + # ADM instead of just those that were in the relevant HEALPixels. + mscx = np.where(inhp & notid)[0][mscx] + log.info('Found {} additional matches...t={:.1f}s'.format( + len(mscx), time()-start)) + + if len(mscx) > 0: + # ADM construct the targetid from the sweeps information. + targetid = encode_targetid(objid=swobjs['OBJID'], + brickid=swobjs['BRICKID'], + release=swobjs['RELEASE']) + # ADM and add the targetid to the secondary targets. + scxtargs["TARGETID"][mscx] = targetid[mswobjs] + # ADM write the secondary targets that have updated TARGETIDs. ii = scxtargs["TARGETID"] != -1 nmatches = np.sum(ii) @@ -730,6 +786,9 @@ def finalize_secondary(scxtargs, scnd_mask, survey='main', sep=1., # on lowest index in list of duplicates dups = np.delete(dups, np.argmax(scxtargs['PRIORITY_INIT'][dups])) alldups.append(dups) + # ADM guard against the case that there are no duplicates. + if len(alldups) == 0: + alldups = [alldups] alldups = np.hstack(alldups) log.debug("Flagging {} duplicate secondary targetids with PRIORITY_INIT=-1".format(len(alldups))) @@ -764,7 +823,8 @@ def finalize_secondary(scxtargs, scnd_mask, survey='main', sep=1., done[pc], done[nc] = initial_priority_numobs(done, obscon=oc, scnd=True) # APC Flagged duplicates are removed in io.write_secondary - done[pc][alldups] = -1 + if len(alldups) > 0: + done[pc][alldups] = -1 # APC add secondary flag in DESI_TARGET cols, mx, surv = main_cmx_or_sv(done, scnd=True) From 3952558929755435c912f0b3590930d6ef814b54 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Wed, 30 Dec 2020 11:07:20 -0800 Subject: [PATCH 02/30] smarter handling of whether a secondary target genuinely matched a primary --- py/desitarget/io.py | 17 +++++++++-------- py/desitarget/secondary.py | 12 +++++++++--- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/py/desitarget/io.py b/py/desitarget/io.py index ac912ff72..33f7ca8f7 100644 --- a/py/desitarget/io.py +++ b/py/desitarget/io.py @@ -924,9 +924,13 @@ def write_secondary(targdir, data, primhdr=None, scxdir=None, obscon=None, np.random.seed(616) data["SUBPRIORITY"] = np.random.random(ntargs) - # ADM remove the SCND_TARGET_INIT and SCND_ORDER columns. - scnd_target_init, scnd_order = data["SCND_TARGET_INIT"], data["SCND_ORDER"] - data = rfn.drop_fields(data, ["SCND_TARGET_INIT", "SCND_ORDER"]) + # ADM remove the SCND_TARGET_INIT, SCND_ORDER and PRIM_MATCH columns. + scnd_target_init = data["SCND_TARGET_INIT"] + scnd_order = data["SCND_ORDER"] + prim_match = data["PRIM_MATCH"] + + data = rfn.drop_fields(data, + ["SCND_TARGET_INIT", "SCND_ORDER", "PRIM_MATCH"]) # ADM we only need a subset of the columns where we match a primary. smalldata = rfn.drop_fields(data, ["PRIORITY_INIT", "SUBPRIORITY", "NUMOBS_INIT", "OBSCONDITIONS"]) @@ -967,11 +971,8 @@ def write_secondary(targdir, data, primhdr=None, scxdir=None, obscon=None, # ADM make necessary directories for the file, if they don't exist. os.makedirs(os.path.dirname(filename), exist_ok=True) - # ADM standalone secondaries have PRIORITY_INIT > -1 and - # ADM release before DR1 (release < 1000). - objid, brickid, release, mock, sky, gaiadr = decode_targetid(data["TARGETID"]) - ii = (release < 1000) & (data["PRIORITY_INIT"] > -1) - + # ADM standalone secondaries don't have PRIM_MATCH set. + ii = ~prim_match # ADM ...write them out. write_with_units(filename, data[ii], extname='SCND_TARGETS', header=hdr) diff --git a/py/desitarget/secondary.py b/py/desitarget/secondary.py index ce91e76c3..b5ed9d795 100644 --- a/py/desitarget/secondary.py +++ b/py/desitarget/secondary.py @@ -91,8 +91,9 @@ # ADM extra columns that are used during processing but are # ADM not an official part of the input or output data model. +# ADM PRIM_MATCH records whether a secondary matches a primary TARGET. suppdatamodel = np.array([], dtype=[ - ('SCND_TARGET_INIT', '>i8'), ('SCND_ORDER', '>i4') + ('SCND_TARGET_INIT', '>i8'), ('SCND_ORDER', '>i4'), ('PRIM_MATCH', '?') ]) @@ -316,6 +317,7 @@ def read_files(scxdir, scnd_mask): scxout["TARGETID"] = -1 scxout["OBSCONDITIONS"] = \ obsconditions.mask(scnd_mask[name].obsconditions) + scxout["PRIM_MATCH"] = False scxall.append(scxout) return np.concatenate(scxall) @@ -393,6 +395,7 @@ def add_primary_info(scxtargs, priminfodir): # ADM now we have the matches, update the secondary targets # ADM with the primary TARGETIDs. scxtargs["TARGETID"][scxii] = primtargs["TARGETID"][primii] + scxtargs["PRIM_MATCH"][scxii] = primtargs["PRIM_MATCH"][primii] # APC Secondary targets that don't match to a primary. # APC all still have TARGETID = -1 at this point. They @@ -557,6 +560,8 @@ def match_secondary(primtargs, scxdir, scndout, sep=1., hipri = np.maximum(targs["PRIORITY_INIT_DARK"], targs["PRIORITY_INIT_BRIGHT"]) scxtargs["PRIORITY_INIT"][mscx] = hipri[mtargs] + # ADM record that we have a match to a primary. + scxtargs["PRIM_MATCH"][mscx] = True # ADM now we're done matching the primary and secondary targets, also # ADM match the secondary targets to sweep files, if passes, to find @@ -723,7 +728,7 @@ def finalize_secondary(scxtargs, scnd_mask, survey='main', sep=1., # ADM check that the objid array was entirely populated. assert np.all(objid != -1) - # ADM assemble the TARGETID, SCND objects have RELEASE==0. + # ADM assemble the TARGETID, SCND objects. targetid = encode_targetid(objid=objid, brickid=brxid, release=release) # ADM a check that the generated TARGETIDs are unique. @@ -734,7 +739,8 @@ def finalize_secondary(scxtargs, scnd_mask, survey='main', sep=1., # ADM assign the unique TARGETIDs to the secondary objects. scxtargs["TARGETID"][nomatch] = targetid[nomatch] - log.debug("Assigned {} targetids to unmatched secondaries".format(len(targetid[nomatch]))) + log.debug("Assigned {} targetids to unmatched secondaries".format( + len(targetid[nomatch]))) # ADM match secondaries to themselves, to ensure duplicates # ADM share a TARGETID. Don't match special (OVERRIDE) targets From 447fcf7fe098ba810250494431bfa3b209d424bb Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Wed, 30 Dec 2020 12:54:17 -0800 Subject: [PATCH 03/30] update output files to better reflect the data model --- py/desitarget/io.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/py/desitarget/io.py b/py/desitarget/io.py index 33f7ca8f7..6c28b1eea 100644 --- a/py/desitarget/io.py +++ b/py/desitarget/io.py @@ -886,13 +886,13 @@ def write_secondary(targdir, data, primhdr=None, scxdir=None, obscon=None, Two sets of files are written: - The file of secondary targets that do not match a primary target is written to `targdir`. Such secondary targets - are determined from having `RELEASE==0` and `SKY==0` - in the `TARGETID`. Only targets with `PRIORITY_INIT > -1` - are written to this file (this allows duplicates to be - resolved in, e.g., :func:`~desitarget.secondary.finalize()` + are determined from having "PRIM_MATCH"=``False`` in `data`. + Only targets with `PRIORITY_INIT > -1` are written to this file + (this allows duplicates to be resolved in, e.g., + :func:`~desitarget.secondary.finalize()`. - Each secondary target that, presumably, was initially drawn from the "indata" subdirectory of `scxdir` is written to - an "outdata/targdir" subdirectory of `scxdir`. + the "outdata" subdirectory of `scxdir`. """ # ADM grab the scxdir, it it wasn't passed. from desitarget.secondary import _get_scxdir @@ -941,14 +941,15 @@ def write_secondary(targdir, data, primhdr=None, scxdir=None, obscon=None, scnd_mask = mx[3] # ADM construct the output full and reduced file name. - filename = find_target_files(targdir, dr=drint, flavor="targets", - survey=survey, obscon=obscon, nohp=True) - filenam = os.path.splitext(os.path.basename(filename))[0] + filename = find_target_files(targdir, dr=drint, flavor="targets", nohp=True, + survey=survey, obscon=obscon, resolve=None) # ADM write out the file of matches for every secondary bit. - scxoutdir = os.path.join(scxdir, 'outdata', filenam) + scxoutdir = os.path.join(scxdir, 'outdata') if obscon is not None: scxoutdir = os.path.join(scxoutdir, obscon.lower()) + else: + scxoutdir = os.path.join(scxoutdir, "no-obscon") os.makedirs(scxoutdir, exist_ok=True) # ADM and write out the information for each bit. @@ -971,8 +972,10 @@ def write_secondary(targdir, data, primhdr=None, scxdir=None, obscon=None, # ADM make necessary directories for the file, if they don't exist. os.makedirs(os.path.dirname(filename), exist_ok=True) - # ADM standalone secondaries don't have PRIM_MATCH set. - ii = ~prim_match + # ADM standalone secondaries have PRIORITY_INIT > -1 and + # ADM don't have PRIM_MATCH set. + ii = ~prim_match & (data["PRIORITY_INIT"] > -1) + # ADM ...write them out. write_with_units(filename, data[ii], extname='SCND_TARGETS', header=hdr) @@ -2163,6 +2166,7 @@ def find_target_files(targdir, dr='X', flavor="targets", survey="main", resolve : :class:`bool`, optional, defaults to ``True`` If ``True`` then find the `resolve` file. Otherwise find the `noresolve` file. Relevant if `flavor` is `targets` or `randoms`. + Pass ``None`` to substitute `resolve` with "secondary". supp : :class:`bool`, optional, defaults to ``False`` If ``True`` then find the supplemental targets file. Overrides the `obscon` option. @@ -2220,8 +2224,11 @@ def find_target_files(targdir, dr='X', flavor="targets", survey="main", log.critical(msg) raise ValueError(msg) res = "noresolve" - if resolve: - res = "resolve" + if resolve is None: + res = "secondary" + else: + if resolve: + res = "resolve" resdir = "" if flavor in ["targets", "randoms"]: resdir = res From 09e0ab337a53fb5cf009ec7b22213bd0ab08f2ca Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 1 Jan 2021 14:30:04 -0800 Subject: [PATCH 04/30] add full bitmask for SV1 secondary targets --- py/desitarget/sv1/data/sv1_targetmask.yaml | 139 ++++++++++++++++++--- 1 file changed, 120 insertions(+), 19 deletions(-) diff --git a/py/desitarget/sv1/data/sv1_targetmask.yaml b/py/desitarget/sv1/data/sv1_targetmask.yaml index 8ba03046d..2f5af22ac 100644 --- a/py/desitarget/sv1/data/sv1_targetmask.yaml +++ b/py/desitarget/sv1/data/sv1_targetmask.yaml @@ -146,17 +146,50 @@ sv1_mws_mask: #- ADM directory that corresponds to the $SECONDARY_DIR/sv1 environment #- ADM variable, e.g. $SECONDARY_DIR/sv1/veto.fits for VETO targets. sv1_scnd_mask: - - [VETO, 0, "Never observe, even if a primary target bit is set", + - [VETO, 0, "Never observe, even if a primary target bit is set", {obsconditions: DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18, filename: 'veto'}] - - [DR14Q, 1, "Known quasars from the SDSS DR14Q catalog", - {obsconditions: DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18, filename: 'dr14q'}] - - [MWS_CALIB, 10, "Stars with APOGEE/BOSS/GALAH/GAIAESO spectra", + - [UDG, 1, "See $SCND_DIR/UDG.txt", {obsconditions: DARK, filename: 'UDG'}] + - [FIRST_MALS, 2, "See $SCND_DIR/FIRST_MALS.txt", {obsconditions: DARK, filename: 'FIRST_MALS'}] + - [WD_BINARIES, 3, "See $SCND_DIR/WD_BINARIES.txt", {obsconditions: DARK, filename: 'WD_BINARIES'}] + - [LBG_TOMOG, 4, "See $SCND_DIR/LBG_TOMOG.txt", {obsconditions: DARK, filename: 'LBG_TOMOG'}] + - [QSO_RED, 5, "See $SCND_DIR/QSO_RED.ipynb", {obsconditions: DARK, filename: 'QSO_RED'}] + - [M31_KNOWN, 6, "See $SCND_DIR/M31_KNOWN.txt", {obsconditions: DARK, filename: 'M31_KNOWN'}] + - [M31_QSO, 7, "See $SCND_DIR/M31_QSO.txt.", {obsconditions: DARK, filename: 'M31_QSO'}] + - [M31_STAR, 8, "See $SCND_DIR/M31_STAR.txt", {obsconditions: DARK, filename: 'M31_STAR'}] +# - [MWS_DDOGIANTS, 9, "See $SCND_DIR", {obsconditions: BRIGHT, filename: 'MWS_DDOGIANTS'}] + - [MWS_CLUS_GAL_DEEP, 10, "See $SCND_DIR/MWS_CLUS_GAL_DEEP.txt", {obsconditions: DARK, filename: 'MWS_CLUS_GAL_DEEP'}] + - [LOW_MASS_AGN, 11, "See $SCND_DIR/LOW_MASS_AGN.txt", {obsconditions: DARK, filename: 'LOW_MASS_AGN'}] + - [FAINT_HPM, 12, "See $SCND_DIR/FAINT_HPM.txt", {obsconditions: DARK, filename: 'FAINT_HPM'}] + - [GW190412, 13, "See $SCND_DIR/GW190412.ipynb", {obsconditions: DARK|BRIGHT, filename: 'GW190412'}] + - [IC134191, 14, "See $SCND_DIR/IC134191.ipynb", {obsconditions: DARK|BRIGHT, filename: 'IC134191'}] + - [PV_BRIGHT, 15, "See $SCND_DIR/PV_BRIGHT.ipynb", {obsconditions: BRIGHT, filename: 'PV_BRIGHT'}] + - [PV_DARK, 16, "See $SCND_DIR/PV_DARK.ipynb", {obsconditions: DARK, filename: 'PV_DARK'}] + - [LOW_Z, 17, "See $SCND_DIR/LOW_Z.ipynb", {obsconditions: DARK, filename: 'LOW_Z'}] + - [BHB, 18, "See $SCND_DIR/BHB.txt", {obsconditions: DARK, filename: 'BHB'}] + - [SPCV, 19, "See $SCND_DIR/SPCV.txt", {obsconditions: DARK, filename: 'SPCV'}] + - [DC3R2_GAMA, 20, "See $SCND_DIR/DC3R2_GAMA.ipynb", {obsconditions: DARK, filename: 'DC3R2_GAMA'}] + - [UNWISE_BLUE, 21, "See $SCND_DIR/UNWISE_BLUE.txt", {obsconditions: DARK, filename: 'UNWISE_BLUE'}] + - [UNWISE_GREEN, 22, "See $SCND_DIR/UNWISE_GREEN.txt", {obsconditions: DARK, filename: 'UNWISE_GREEN'}] + - [HETDEX_MAIN, 23, "See $SCND_DIR/HETDEX_MAIN.txt", {obsconditions: DARK, filename: 'HETDEX_MAIN'}] + - [HETDEX_HP, 24, "See $SCND_DIR/HETDEX_HP.txt", {obsconditions: DARK, filename: 'HETDEX_HP'}] +# - [PSF_OUT_BRIGHT, 25, "See $SCND_DIR", {obsconditions: BRIGHT, filename: 'PSF_OUT_BRIGHT'}] +# - [PSF_OUT_DARK, 26, "See $SCND_DIR", {obsconditions: DARK, filename: 'PSF_OUT_DARK'}] + - [HPM_SOUM, 27, "See $SCND_DIR/HPM_SOUM.txt", {obsconditions: DARK, filename: 'HPM_SOUM'}] + - [SN_HOSTS, 28, "See $SCND_DIR/SN_HOSTS.txt", {obsconditions: DARK, filename: 'SN_HOSTS'}] + - [GAL_CLUS_BCG, 29, "See $SCND_DIR/GAL_CLUS_BCG.txt", {obsconditions: DARK, filename: 'GAL_CLUS_BCG'}] + - [GAL_CLUS_2ND, 30, "See $SCND_DIR/GAL_CLUS_2ND.txt", {obsconditions: DARK, filename: 'GAL_CLUS_2ND'}] + - [GAL_CLUS_SAT, 31, "See $SCND_DIR/GAL_CLUS_SAT.txt", {obsconditions: DARK, filename: 'GAL_CLUS_SAT'}] + - [HSC_HIZ_SNE, 32, "See $SCND_DIR/HSC_HIZ_SNE.txt", {obsconditions: DARK, filename: 'HSC_HIZ_SNE'}] +# - [ISM_CGM_QGP, 33, "See $SCND_DIR", {obsconditions: DARK, filename: 'ISM_CGM_QGP'}] + - [STRONG_LENS, 34, "See $SCND_DIR/STRONG_LENS.txt", {obsconditions: DARK, filename: 'STRONG_LENS'}] + - [WISE_VAR_QSO, 35, "See $SCND_DIR/WISE_VAR_QSO.txt", {obsconditions: DARK, filename: 'WISE_VAR_QSO'}] + - [MWS_CALIB, 36, "Stars with APOGEE/BOSS/GALAH/GAIAESO spectra", {obsconditions: DARK|GRAY|BRIGHT, filename: 'MWS_CALIB'}] - - [BACKUP_CALIB, 11, "Very bright stars with APOGEE/BOSS/GALAH/GAIAESO spectra", + - [BACKUP_CALIB, 37, "Very bright stars with APOGEE/BOSS/GALAH/GAIAESO spectra", {obsconditions: POOR|TWILIGHT12|TWILIGHT18, filename: 'BACKUP_CALIB'}] - - [MWS_MAIN_CLUSTER_SV, 13, "Main survey low mass and metal poor stars (OC/GC/dwarf/stream members)", + - [MWS_MAIN_CLUSTER_SV, 38, "Main survey low mass and metal poor stars (OC/GC/dwarf/stream members)", {obsconditions: DARK|GRAY|BRIGHT, filename: 'MWS_MAIN_CLUSTER_SV'}] - - [MWS_MAIN_RRLYR, 14, "Main survey halo tracers (RR Lyrae)", + - [MWS_RRLYR, 39, "Main survey halo tracers (RR Lyrae)", {obsconditions: DARK|GRAY|BRIGHT, filename: 'MWS_MAIN_RRLYR'}] #- Observation State @@ -304,12 +337,46 @@ priorities: # ADM secondary target priorities. Probably all have very low UNOBS sv1_scnd_mask: - VETO: {UNOBS: 0, DONE: 0, OBS: 0, DONOTOBSERVE: 0} - DR14Q: {UNOBS: 10, DONE: 2, OBS: 1, DONOTOBSERVE: 0} - MWS_CALIB: {UNOBS: 2995, DONE: 2, OBS: 1, DONOTOBSERVE: 0} - BACKUP_CALIB: SAME_AS_MWS_CALIB - MWS_MAIN_CLUSTER_SV: {UNOBS: 1450, DONE: 400, OBS: 1, DONOTOBSERVE: 0} - MWS_MAIN_RRLYR: {UNOBS: 1450, DONE: 400, OBS: 1, DONOTOBSERVE: 0} + VETO: {UNOBS: 0, DONE: 0, OBS: 0, DONOTOBSERVE: 0} + UDG: {UNOBS: 1900, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + FIRST_MALS: {UNOBS: 1025, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + WD_BINARIES: {UNOBS: 1998, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + LBG_TOMOG: {UNOBS: 1800, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + QSO_RED: {UNOBS: 3400, MORE_ZGOOD: 3350, MORE_ZWARN: 3300, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + M31_KNOWN: SAME_AS_LBG_TOMOG + M31_QSO: SAME_AS_LBG_TOMOG + M31_STAR: SAME_AS_LBG_TOMOG + MWS_DDOGIANTS: {UNOBS: 1450, DONE: 2, OBS: 1, DONOTOBSERVE: 0 + MWS_CLUS_GAL_DEEP: SAME_AS_MWS_DDOGIANTS + LOW_MASS_AGN: {UNOBS: 1025, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + FAINT_HPM: SAME_AS_LOW_MASS_AGN + GW190412: {UNOBS: 4000, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + IC134191: SAME_AS_GW190412 + PV_BRIGHT: {UNOBS: 100, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + PV_DARK: {UNOBS: 100, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + LOW_Z: {UNOBS: 90, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + BHB: {UNOBS: 1950, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + SPCV: SAME_AS_LOW_MASS_AGN + DC3R2_GAMA: SAME_AS_LOW_MASS_AGN + UNWISE_BLUE: SAME_AS_LBG_TOMOG + UNWISE_GREEN: SAME_AS_LBG_TOMOG + HETDEX_MAIN: SAME_AS_LBG_TOMOG + HETDEX_HP: SAME_AS_LBG_TOMOG +# PSF_OUT_BRIGHT: {UNOBS: 70, DONE: 2, OBS: 1, DONOTOBSERVE: 0} +# PSF_OUT_DARK: SAME_AS_PSF_OUT_BRIGHT + HPM_SOUM: SAME_AS_LOW_MASS_AGN + SN_HOSTS: SAME_AS_BHB + GAL_CLUS_BCG: {UNOBS: 1025, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + GAL_CLUS_2ND: {UNOBS: 1024, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + GAL_CLUS_SAT: {UNOBS: 200, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + HSC_HIZ_SNE: SAME_AS_LBG_TOMOG +# ISM_CGM_QGP: {UNOBS: 4000, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + STRONG_LENS: SAME_AS_UDG + WISE_VAR_QSO: SAME_AS_QSO_RED + MWS_CALIB: {UNOBS: 2995, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + BACKUP_CALIB: SAME_AS_MWS_CALIB + MWS_MAIN_CLUSTER_SV: {UNOBS: 1450, DONE: 400, OBS: 1, DONOTOBSERVE: 0} + MWS_MAIN_RRLYR: {UNOBS: 1450, DONE: 400, OBS: 1, DONOTOBSERVE: 0} # ADM INITIAL number of observations (NUMOBS) for each target bit # ADM SAME_AS_XXX means to use the NUMOBS for bitname XXX @@ -427,9 +494,43 @@ numobs: # ADM initial number of observations for secondary targets sv1_scnd_mask: - VETO: 1 - DR14Q: 1 - MWS_CALIB: 100 - MWS_MAIN_CLUSTER_SV: 1 - MWS_MAIN_RRLYR: 100 - BACKUP_CALIB: SAME_AS_MWS_CALIB + VETO: 1 + UDG: 100 + FIRST_MALS: 100 + WD_BINARIES: 100 + LBG_TOMOG: 100 + QSO_RED: 100 + M31_KNOWN: 100 + M31_QSO: 100 + M31_STAR: 100 + MWS_DDOGIANTS: 100 + MWS_CLUS_GAL_DEEP: 100 + LOW_MASS_AGN: 100 + FAINT_HPM: 100 + GW190412: 100 + IC134191: 100 + PV_BRIGHT: 100 + PV_DARK: 100 + LOW_Z: 100 + BHB: 100 + SPCV: 100 + DC3R2_GAMA: 100 + UNWISE_BLUE: 100 + UNWISE_GREEN: 100 + HETDEX_MAIN: 100 + HETDEX_HP: 100 + PSF_OUT_BRIGHT: 100 + PSF_OUT_DARK: 100 + HPM_SOUM: 100 + SN_HOSTS: 100 + GAL_CLUS_BCG: 100 + GAL_CLUS_2ND: 100 + GAL_CLUS_SAT: 100 + HSC_HIZ_SNE: 100 + ISM_CGM_QGP: 100 + STRONG_LENS: 100 + WISE_VAR_QSO: 100 + MWS_CALIB: 100 + MWS_MAIN_CLUSTER_SV: 1 + MWS_MAIN_RRLYR: 100 + BACKUP_CALIB: SAME_AS_MWS_CALIB From 0ca8bd55e0b4c366f942ecd3b5bb5de29c88abc4 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 1 Jan 2021 14:44:13 -0800 Subject: [PATCH 05/30] clean up syntax errors in yaml file --- py/desitarget/sv1/data/sv1_targetmask.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/py/desitarget/sv1/data/sv1_targetmask.yaml b/py/desitarget/sv1/data/sv1_targetmask.yaml index 2f5af22ac..00d116758 100644 --- a/py/desitarget/sv1/data/sv1_targetmask.yaml +++ b/py/desitarget/sv1/data/sv1_targetmask.yaml @@ -190,7 +190,7 @@ sv1_scnd_mask: - [MWS_MAIN_CLUSTER_SV, 38, "Main survey low mass and metal poor stars (OC/GC/dwarf/stream members)", {obsconditions: DARK|GRAY|BRIGHT, filename: 'MWS_MAIN_CLUSTER_SV'}] - [MWS_RRLYR, 39, "Main survey halo tracers (RR Lyrae)", - {obsconditions: DARK|GRAY|BRIGHT, filename: 'MWS_MAIN_RRLYR'}] + {obsconditions: DARK|GRAY|BRIGHT, filename: 'MWS_RRLYR'}] #- Observation State #- if a target passes more than one target bit, it is possible that one bit @@ -346,8 +346,8 @@ priorities: M31_KNOWN: SAME_AS_LBG_TOMOG M31_QSO: SAME_AS_LBG_TOMOG M31_STAR: SAME_AS_LBG_TOMOG - MWS_DDOGIANTS: {UNOBS: 1450, DONE: 2, OBS: 1, DONOTOBSERVE: 0 - MWS_CLUS_GAL_DEEP: SAME_AS_MWS_DDOGIANTS +# MWS_DDOGIANTS: {UNOBS: 1450, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + MWS_CLUS_GAL_DEEP: {UNOBS: 1450, DONE: 2, OBS: 1, DONOTOBSERVE: 0} LOW_MASS_AGN: {UNOBS: 1025, DONE: 2, OBS: 1, DONOTOBSERVE: 0} FAINT_HPM: SAME_AS_LOW_MASS_AGN GW190412: {UNOBS: 4000, DONE: 2, OBS: 1, DONOTOBSERVE: 0} @@ -376,7 +376,7 @@ priorities: MWS_CALIB: {UNOBS: 2995, DONE: 2, OBS: 1, DONOTOBSERVE: 0} BACKUP_CALIB: SAME_AS_MWS_CALIB MWS_MAIN_CLUSTER_SV: {UNOBS: 1450, DONE: 400, OBS: 1, DONOTOBSERVE: 0} - MWS_MAIN_RRLYR: {UNOBS: 1450, DONE: 400, OBS: 1, DONOTOBSERVE: 0} + MWS_RRLYR: {UNOBS: 1450, DONE: 400, OBS: 1, DONOTOBSERVE: 0} # ADM INITIAL number of observations (NUMOBS) for each target bit # ADM SAME_AS_XXX means to use the NUMOBS for bitname XXX @@ -503,7 +503,7 @@ numobs: M31_KNOWN: 100 M31_QSO: 100 M31_STAR: 100 - MWS_DDOGIANTS: 100 +# MWS_DDOGIANTS: 100 MWS_CLUS_GAL_DEEP: 100 LOW_MASS_AGN: 100 FAINT_HPM: 100 @@ -519,18 +519,18 @@ numobs: UNWISE_GREEN: 100 HETDEX_MAIN: 100 HETDEX_HP: 100 - PSF_OUT_BRIGHT: 100 - PSF_OUT_DARK: 100 +# PSF_OUT_BRIGHT: 100 +# PSF_OUT_DARK: 100 HPM_SOUM: 100 SN_HOSTS: 100 GAL_CLUS_BCG: 100 GAL_CLUS_2ND: 100 GAL_CLUS_SAT: 100 HSC_HIZ_SNE: 100 - ISM_CGM_QGP: 100 +# ISM_CGM_QGP: 100 STRONG_LENS: 100 WISE_VAR_QSO: 100 MWS_CALIB: 100 MWS_MAIN_CLUSTER_SV: 1 - MWS_MAIN_RRLYR: 100 + MWS_RRLYR: 100 BACKUP_CALIB: SAME_AS_MWS_CALIB From 90cf10d8759750400c833bf6431b2e5a16ed6803 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Sat, 2 Jan 2021 08:55:09 -0800 Subject: [PATCH 06/30] clean up the data model, although secondary files to be .csv as well as .txt --- bin/select_secondary | 2 +- bin/select_sv_targets | 2 +- py/desitarget/secondary.py | 27 ++++++++++++++++++---- py/desitarget/sv1/data/sv1_targetmask.yaml | 2 +- 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/bin/select_secondary b/bin/select_secondary index 7a8c2756a..16de05f0e 100755 --- a/bin/select_secondary +++ b/bin/select_secondary @@ -43,7 +43,7 @@ hdr = fitsio.read_header(fns[0], 'SCND_TARG') surv = hdr["SURVEY"].rstrip() # ADM find the SCND_DIR environment variable, if it wasn't passed. -scxdir = _get_scxdir(ns.scnddir) +scxdir = _get_scxdir(ns.scnddir, survey=surv) # ADM and augment the scxdir if this is an SV set of primary files. if surv != 'main': scxdir = os.path.join(scxdir, surv) diff --git a/bin/select_sv_targets b/bin/select_sv_targets index 349375e02..52c169b8c 100755 --- a/bin/select_sv_targets +++ b/bin/select_sv_targets @@ -161,7 +161,7 @@ if ns.bundlefiles is None: if not ns.nosecondary and len(targets) > 0: from desitarget.secondary import _get_scxdir, match_secondary # ADM read secondary target directory. - scxdir = _get_scxdir(ns.scnddir) + scxdir = _get_scxdir(ns.scnddir, survey=survey) # ADM construct a label for the secondary file for TARGETID look-ups. # ADM use RELEASE to determine the release string for the targets. drint = int(np.max(targets['RELEASE']//1000)) diff --git a/py/desitarget/secondary.py b/py/desitarget/secondary.py index b5ed9d795..dc7e7ae59 100644 --- a/py/desitarget/secondary.py +++ b/py/desitarget/secondary.py @@ -127,7 +127,7 @@ def duplicates(seq): return ((key, np.array(locs)) for key, locs in tally.items() if len(locs) > 1) -def _get_scxdir(scxdir=None): +def _get_scxdir(scxdir=None, survey=""): """Retrieve the base secondary directory with error checking. Parameters @@ -136,6 +136,9 @@ def _get_scxdir(scxdir=None): Directory containing secondary target files to which to match. If not specified, the directory is taken to be the value of the :envvar:`SCND_DIR` environment variable. + survey : :class:`str`, optional, defaults to "" for the Main Survey. + Flavor of survey that we're processing, e.g., "sv1". Don't pass + anything for "main" (the Main Survey). Returns ------- @@ -155,7 +158,8 @@ def _get_scxdir(scxdir=None): # ADM also fail if the indata, outdata and docs directories don't # ADM exist in the secondary directory. - for subdir in "docs", "indata", "outdata": + checkdir = [os.path.join(survey, d) for d in ["docs", "indata", "outdata"]] + for subdir in checkdir: if not os.path.isdir(os.path.join(scxdir, subdir)): msg = '{} directory not found in {}'.format(subdir, scxdir) log.critical(msg) @@ -285,8 +289,20 @@ def read_files(scxdir, scnd_mask): log.debug(' path: {}'.format(fn)) # ADM if the relevant file is a .txt file, read it in. if os.path.exists(fn+'.txt'): - scxin = np.loadtxt(fn+'.txt', usecols=[0, 1, 2, 3, 4, 5], - dtype=indatamodel.dtype) + try: + scxin = np.loadtxt(fn+'.txt', usecols=[0, 1, 2, 3, 4, 5], + dtype=indatamodel.dtype) + except (ValueError, IndexError): + msg = "First 6 columns don't correspond to {} in {}.txt".format( + indatamodel.dtype, fn) + # ADM perhaps people provided .csv files as .txt files. + try: + scxin = np.loadtxt(fn+'.txt', usecols=[0, 1, 2, 3, 4, 5], + dtype=indatamodel.dtype, delimiter=",") + except (ValueError, IndexError): + log.error(msg) + raise IOError(msg) + # ADM otherwise it's a fits file, read it in. else: scxin = fitsio.read(fn+'.fits', @@ -296,7 +312,8 @@ def read_files(scxdir, scnd_mask): scxin = np.atleast_1d(scxin) # ADM assert the data model. - msg = "Data model doesn't match {} in {}".format(indatamodel.dtype, fn) + msg = "Data model doesn't match {} in {}.fits".format( + indatamodel.dtype, fn) for col in indatamodel.dtype.names: assert scxin[col].dtype == indatamodel[col].dtype, msg diff --git a/py/desitarget/sv1/data/sv1_targetmask.yaml b/py/desitarget/sv1/data/sv1_targetmask.yaml index 00d116758..f32e4306f 100644 --- a/py/desitarget/sv1/data/sv1_targetmask.yaml +++ b/py/desitarget/sv1/data/sv1_targetmask.yaml @@ -144,7 +144,7 @@ sv1_mws_mask: #- ADM SV secondary survey targets bit mask. #- ADM note that the 'filenames', here, should all be present in the #- ADM directory that corresponds to the $SECONDARY_DIR/sv1 environment -#- ADM variable, e.g. $SECONDARY_DIR/sv1/veto.fits for VETO targets. +#- ADM variable, e.g. $SECONDARY_DIR/sv1/veto.txt for VETO targets. sv1_scnd_mask: - [VETO, 0, "Never observe, even if a primary target bit is set", {obsconditions: DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18, filename: 'veto'}] From 10d07446af656ab59761f36dcc8aaa1b9e25f709 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Sat, 2 Jan 2021 14:05:08 -0800 Subject: [PATCH 07/30] general clean up of output/input data model and logging --- bin/select_sv_targets | 18 ++++++++++-------- bin/select_targets | 18 ++++++++++-------- py/desitarget/io.py | 2 +- py/desitarget/secondary.py | 9 +++++---- 4 files changed, 26 insertions(+), 21 deletions(-) diff --git a/bin/select_sv_targets b/bin/select_sv_targets index 52c169b8c..a4f468533 100755 --- a/bin/select_sv_targets +++ b/bin/select_sv_targets @@ -67,7 +67,7 @@ ap.add_argument("--nomaskbits", action='store_true', ap.add_argument("--writeall", action='store_true', help="Default behavior is to split targets by bright/dark-time surveys. Send this to ALSO write a file of ALL targets") ap.add_argument("-nos", "--nosecondary", action='store_true', - help="Do NOT create TARGETID look-up files for secondary targets in $SCNDIR/outdata/priminfo-drversion-desitargetversion/$dest.fits (where $dest is the basename of dest)") + help="Do NOT create TARGETID look-up files for secondary targets in $SCNDIR/outdata/desitargetversion/priminfo-drversion-desitargetversion/$dest.fits (where $dest is the basename of dest)") ap.add_argument("--scnddir", help="Base directory of secondary target files (e.g. '/project/projectdirs/desi/target/secondary' at NERSC). "+ "Defaults to SCND_DIR environment variable. Not needed if --nosecondary is sent.") @@ -148,14 +148,15 @@ targets, infn = select_targets( tcnames=tcnames, survey=survey, backup=not(ns.nobackup), resolvetargs=not(ns.noresolve), mask=not(ns.nomaskbits), return_infiles=True ) -# ADM Set the list of infiles actually processed by select_targets() to -# ADM None if we DON'T want to write their checksums to the output file. -if ns.nochecksum: - shatab = None -else: - shatab = get_checksums(infn, verbose=True) if ns.bundlefiles is None: + # ADM Set the list of infiles actually processed by select_targets() to + # ADM None if we DON'T want to write their checksums to the output file. + if ns.nochecksum: + shatab = None + else: + shatab = get_checksums(infn, verbose=True) + # ADM only run secondary functions if --nosecondary was not passed. scndout = None if not ns.nosecondary and len(targets) > 0: @@ -166,7 +167,8 @@ if ns.bundlefiles is None: # ADM use RELEASE to determine the release string for the targets. drint = int(np.max(targets['RELEASE']//1000)) scndoutdn = "priminfo-dr{}-{}".format(drint, desitarget_version) - scndoutdn = os.path.join(scxdir, survey, "outdata", scndoutdn) + scndoutdn = os.path.join(scxdir, survey, + "outdata", desitarget_version, scndoutdn) if not os.path.exists(scndoutdn): log.info("making directory...{}".format(scndoutdn)) os.makedirs(scndoutdn) diff --git a/bin/select_targets b/bin/select_targets index b797d24d3..870b8c074 100755 --- a/bin/select_targets +++ b/bin/select_targets @@ -69,7 +69,7 @@ ap.add_argument("--nomaskbits", action='store_true', ap.add_argument("--writeall", action='store_true', help="Default behavior is to split targets by bright/dark-time surveys. Send this to ALSO write a file of ALL targets") ap.add_argument("-nos", "--nosecondary", action='store_true', - help="Do NOT create TARGETID look-up files for secondary targets in $SCNDIR/outdata/priminfo-drversion-desitargetversion/$dest.fits (where $dest is the basename of dest)") + help="Do NOT create TARGETID look-up files for secondary targets in $SCNDIR/outdata/desitargetversion/priminfo-drversion-desitargetversion/$dest.fits (where $dest is the basename of dest)") ap.add_argument("--scnddir", help="Base directory of secondary target files (e.g. '/project/projectdirs/desi/target/secondary' at NERSC). "+ "Defaults to SCND_DIR environment variable. Not needed if --nosecondary is sent.") @@ -149,14 +149,15 @@ targets, infn = select_targets( tcnames=tcnames, survey='main', backup=not(ns.nobackup), resolvetargs=not(ns.noresolve), mask=not(ns.nomaskbits), return_infiles=True ) -# ADM Set the list of infiles actually processed by select_targets() to -# ADM None if we DON'T want to write their checksums to the output file. -if ns.nochecksum: - shatab = None -else: - shatab = get_checksums(infn, verbose=True) if ns.bundlefiles is None: + # ADM Set the list of infiles actually processed by select_targets() to + # ADM None if we DON'T want to write their checksums to the output file. + if ns.nochecksum: + shatab = None + else: + shatab = get_checksums(infn, verbose=True) + # ADM only run secondary functions if --nosecondary was not passed. scndout = None if not ns.nosecondary and len(targets) > 0: @@ -167,7 +168,8 @@ if ns.bundlefiles is None: # ADM use RELEASE to determine the release string for the targets. drint = int(np.max(targets['RELEASE']//1000)) scndoutdn = "priminfo-dr{}-{}".format(drint, desitarget_version) - scndoutdn = os.path.join(scxdir, "outdata", scndoutdn) + scndoutdn = os.path.join(scxdir, + "outdata", desitarget_version, scndoutdn) if not os.path.exists(scndoutdn): log.info("making directory...{}".format(scndoutdn)) os.makedirs(scndoutdn) diff --git a/py/desitarget/io.py b/py/desitarget/io.py index 6c28b1eea..d7a7dfece 100644 --- a/py/desitarget/io.py +++ b/py/desitarget/io.py @@ -945,7 +945,7 @@ def write_secondary(targdir, data, primhdr=None, scxdir=None, obscon=None, survey=survey, obscon=obscon, resolve=None) # ADM write out the file of matches for every secondary bit. - scxoutdir = os.path.join(scxdir, 'outdata') + scxoutdir = os.path.join(scxdir, 'outdata', desitarget_version) if obscon is not None: scxoutdir = os.path.join(scxoutdir, obscon.lower()) else: diff --git a/py/desitarget/secondary.py b/py/desitarget/secondary.py index dc7e7ae59..1670606c1 100644 --- a/py/desitarget/secondary.py +++ b/py/desitarget/secondary.py @@ -205,8 +205,9 @@ def _check_files(scxdir, scnd_mask): # ADM extensions in each of the indata and docs directories. setdic = {} for subdir in 'indata', 'docs': - # ADM retrieve the full file names. - fnswext = os.listdir(os.path.join(scxdir, subdir)) + # ADM retrieve the full file names. Ignore directories. + fnswext = [fn for fn in os.listdir(os.path.join(scxdir, subdir)) if + os.path.isfile(os.path.join(scxdir, subdir, fn))] # ADM split off the extensions. exts = [os.path.splitext(fn)[1] for fn in fnswext] # ADM check they're all allowed extensions. @@ -764,8 +765,8 @@ def finalize_secondary(scxtargs, scnd_mask, survey='main', sep=1., # ADM or sources that have already been matched to a primary. w = np.where(~scxtargs["OVERRIDE"] & nomatch)[0] if len(w) > 0: - log.info("Matching secondary targets to themselves...t={:.1f}s" - .format(time()-t0)) + log.info("Matching {} secondary targets to themselves...t={:.1f}s" + .format(len(scxtargs), time()-t0)) # ADM use astropy for the matching. At NERSC, astropy matches # ADM ~20M objects to themselves in about 10 minutes. c = SkyCoord(scxtargs["RA"][w]*u.deg, scxtargs["DEC"][w]*u.deg) From f5ce532ae7c0d6f424ab9261b1c708e47839834d Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Sat, 2 Jan 2021 14:23:41 -0800 Subject: [PATCH 08/30] allow the secondary directory to be passed to the slurm/bundling script --- bin/select_sv_targets | 13 +++++++------ bin/select_targets | 13 +++++++------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/bin/select_sv_targets b/bin/select_sv_targets index a4f468533..6f1dbc74b 100755 --- a/bin/select_sv_targets +++ b/bin/select_sv_targets @@ -80,13 +80,14 @@ ns = ap.parse_args() # ADM build the list of command line arguments as # ADM bundlefiles potentially needs to know about them. extra = " --numproc {}".format(ns.numproc) -if ns.tcnames is not None: - extra += " --tcnames {}".format(ns.tcnames) nsdict = vars(ns) -for nskey in ["noresolve", "nomaskbits", "writeall", - "nosecondary", "nobackup", "nochecksum"]: - if nsdict[nskey]: - extra += " --{}".format(nskey) +for nskey in ["tcnames", "noresolve", "nomaskbits", "writeall", + "nosecondary", "nobackup", "nochecksum", "scnddir"]: + if isinstance(nsdict[nskey], bool): + if nsdict[nskey]: + extra += " --{}".format(nskey) + elif nsdict[nskey] is not None: + extra += " --{} {}".format(nskey, nsdict[nskey]) infiles = io.list_sweepfiles(ns.sweepdir) if ns.sweepdir2 is not None: diff --git a/bin/select_targets b/bin/select_targets index 870b8c074..5cd42fb23 100755 --- a/bin/select_targets +++ b/bin/select_targets @@ -82,13 +82,14 @@ ns = ap.parse_args() # ADM build the list of command line arguments as # ADM bundlefiles potentially needs to know about them. extra = " --numproc {}".format(ns.numproc) -if ns.tcnames is not None: - extra += " --tcnames {}".format(ns.tcnames) nsdict = vars(ns) -for nskey in ["noresolve", "nomaskbits", "writeall", - "nosecondary", "nobackup", "nochecksum"]: - if nsdict[nskey]: - extra += " --{}".format(nskey) +for nskey in ["tcnames", "noresolve", "nomaskbits", "writeall", + "nosecondary", "nobackup", "nochecksum", "scnddir"]: + if isinstance(nsdict[nskey], bool): + if nsdict[nskey]: + extra += " --{}".format(nskey) + elif nsdict[nskey] is not None: + extra += " --{} {}".format(nskey, nsdict[nskey]) infiles = io.list_sweepfiles(ns.sweepdir) if ns.sweepdir2 is not None: From d0a0b7ff9961d44d0e8f0aca073fe27a65066011 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Sat, 2 Jan 2021 15:17:45 -0800 Subject: [PATCH 09/30] make sure the files of matches to primary sources are all written to the same DR directory --- bin/select_sv_targets | 6 ++++-- bin/select_targets | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/bin/select_sv_targets b/bin/select_sv_targets index 6f1dbc74b..b36602442 100755 --- a/bin/select_sv_targets +++ b/bin/select_sv_targets @@ -165,8 +165,10 @@ if ns.bundlefiles is None: # ADM read secondary target directory. scxdir = _get_scxdir(ns.scnddir, survey=survey) # ADM construct a label for the secondary file for TARGETID look-ups. - # ADM use RELEASE to determine the release string for the targets. - drint = int(np.max(targets['RELEASE']//1000)) + try: + drint = int(ns.sweepdir.split("dr")[1][0]) + except (ValueError, IndexError, AttributeError): + drint = "X" scndoutdn = "priminfo-dr{}-{}".format(drint, desitarget_version) scndoutdn = os.path.join(scxdir, survey, "outdata", desitarget_version, scndoutdn) diff --git a/bin/select_targets b/bin/select_targets index 5cd42fb23..0e5f0aa90 100755 --- a/bin/select_targets +++ b/bin/select_targets @@ -166,8 +166,10 @@ if ns.bundlefiles is None: # ADM read secondary target directory. scxdir = _get_scxdir(ns.scnddir) # ADM construct a label for the secondary file for TARGETID look-ups. - # ADM use RELEASE to determine the release string for the targets. - drint = int(np.max(targets['RELEASE']//1000)) + try: + drint = int(ns.sweepdir.split("dr")[1][0]) + except (ValueError, IndexError, AttributeError): + drint = "X" scndoutdn = "priminfo-dr{}-{}".format(drint, desitarget_version) scndoutdn = os.path.join(scxdir, "outdata", desitarget_version, scndoutdn) From a65a2a5ba3488ee99a1780e71848e1fb2a7856a0 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Sun, 3 Jan 2021 08:28:08 -0800 Subject: [PATCH 10/30] debug case where there are no in-pixel sweeps sources --- py/desitarget/secondary.py | 47 ++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/py/desitarget/secondary.py b/py/desitarget/secondary.py index 1670606c1..9ae8427d5 100644 --- a/py/desitarget/secondary.py +++ b/py/desitarget/secondary.py @@ -603,28 +603,31 @@ def match_secondary(primtargs, scxdir, scndout, sep=1., swobjs = np.concatenate(swobjs) log.info("Total sources read: {}".format(len(swobjs))) - # ADM limit to just secondary targets in the healpix of interest. - inhp = np.ones(len(scxtargs), dtype="?") - if nside is not None and pix is not None: - inhp = is_in_hp(scxtargs, nside, pix) - - # ADM now perform the match. - log.info('Matching secondary targets to input sweep files...t={:.1f}s' - .format(time()-start)) - mswobjs, mscx = radec_match_to(swobjs, scxtargs[inhp & notid], sep=sep) - # ADM recast the indices to the full set of secondary targets, - # ADM instead of just those that were in the relevant HEALPixels. - mscx = np.where(inhp & notid)[0][mscx] - log.info('Found {} additional matches...t={:.1f}s'.format( - len(mscx), time()-start)) - - if len(mscx) > 0: - # ADM construct the targetid from the sweeps information. - targetid = encode_targetid(objid=swobjs['OBJID'], - brickid=swobjs['BRICKID'], - release=swobjs['RELEASE']) - # ADM and add the targetid to the secondary targets. - scxtargs["TARGETID"][mscx] = targetid[mswobjs] + # ADM continue if there are sources in the pixels of interest. + if len(swobjs) > 0: + # ADM limit to just secondaries in the healpix of interest. + inhp = np.ones(len(scxtargs), dtype="?") + if nside is not None and pix is not None: + inhp = is_in_hp(scxtargs, nside, pix) + + # ADM now perform the match. + log.info('Matching secondary targets to sweep files...t={:.1f}s' + .format(time()-start)) + mswobjs, mscx = radec_match_to(swobjs, + scxtargs[inhp & notid], sep=sep) + # ADM recast the indices to the full set of secondaries, + # ADM instead of just those that were in the relevant pixels. + mscx = np.where(inhp & notid)[0][mscx] + log.info('Found {} additional matches...t={:.1f}s'.format( + len(mscx), time()-start)) + + if len(mscx) > 0: + # ADM construct the targetid from the sweeps information. + targetid = encode_targetid(objid=swobjs['OBJID'], + brickid=swobjs['BRICKID'], + release=swobjs['RELEASE']) + # ADM and add the targetid to the secondary targets. + scxtargs["TARGETID"][mscx] = targetid[mswobjs] # ADM write the secondary targets that have updated TARGETIDs. ii = scxtargs["TARGETID"] != -1 From e8f960a2447d5a2185114158ee199938a1728253 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Mon, 4 Jan 2021 00:43:43 -0800 Subject: [PATCH 11/30] quicker reading across the pixelized files of primary information --- py/desitarget/secondary.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/py/desitarget/secondary.py b/py/desitarget/secondary.py index 9ae8427d5..ccfbb6ab7 100644 --- a/py/desitarget/secondary.py +++ b/py/desitarget/secondary.py @@ -369,10 +369,14 @@ def add_primary_info(scxtargs, priminfodir): log.warning("No secondary target matches a primary target!!!") return scxtargs - primtargs = fitsio.read(primfns[0]) - for i in range(1, len(primfns)): - prim = fitsio.read(primfns[i]) - primtargs = np.concatenate([primtargs, prim]) + log.info("Begin reading files from {}...t={:.1f}s".format( + priminfodir, time()-start)) + primtargs = [] + for primfn in primfns: + prim = fitsio.read(primfn) + primtargs.append(prim) + primtargs = np.concatenate(primtargs) + log.info("Done reading files...t={:.1f}s".format(time()-start)) # ADM make a unique look-up for the target sets. scxbitnum = np.log2(scxtargs["SCND_TARGET"]).astype('int') @@ -415,7 +419,7 @@ def add_primary_info(scxtargs, priminfodir): scxtargs["TARGETID"][scxii] = primtargs["TARGETID"][primii] scxtargs["PRIM_MATCH"][scxii] = primtargs["PRIM_MATCH"][primii] - # APC Secondary targets that don't match to a primary. + # APC Secondary targets that don't match to a primary target. # APC all still have TARGETID = -1 at this point. They # APC get removed in finalize_secondary(). log.info("Done matching primaries in {} to secondaries...t={:.1f}s" From 57829bfa467aa418b64d718c2c5d41a5f0a8d73d Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Mon, 4 Jan 2021 16:35:15 -0800 Subject: [PATCH 12/30] debug resolving various matching cases: A primary can match multiple secondaries; a secondary can match multiple primaries --- py/desitarget/secondary.py | 44 +++++++++++++++++----- py/desitarget/sv1/data/sv1_targetmask.yaml | 16 ++++---- py/desitarget/targets.py | 8 +++- 3 files changed, 48 insertions(+), 20 deletions(-) diff --git a/py/desitarget/secondary.py b/py/desitarget/secondary.py index ccfbb6ab7..3085522e9 100644 --- a/py/desitarget/secondary.py +++ b/py/desitarget/secondary.py @@ -56,7 +56,7 @@ from desitarget.internal import sharedmem from desitarget.geomask import radec_match_to, add_hp_neighbors, is_in_hp -from desitarget.targets import encode_targetid, main_cmx_or_sv +from desitarget.targets import encode_targetid, main_cmx_or_sv, resolve from desitarget.targets import set_obsconditions, initial_priority_numobs from desitarget.targetmask import obsconditions @@ -381,16 +381,37 @@ def add_primary_info(scxtargs, priminfodir): # ADM make a unique look-up for the target sets. scxbitnum = np.log2(scxtargs["SCND_TARGET"]).astype('int') primbitnum = np.log2(primtargs["SCND_TARGET"]).astype('int') - - scxids = 1000 * scxtargs["SCND_ORDER"] + scxbitnum - primids = 1000 * primtargs["SCND_ORDER"] + primbitnum - - # ADM if a secondary matched TWO (or more) primaries, - # ADM only retain the highest-priority primary. + # ADM SCND_ORDER can potentially run into the tens-of-millions, so + # ADM we need to use int64 type to get as high as 1000 x SCND_ORDER. + scxids = 1000 * scxtargs["SCND_ORDER"].astype('int64') + scxbitnum + primids = 1000 * primtargs["SCND_ORDER"].astype('int64') + primbitnum + + # ADM a primary (with the same TARGETID) could match TWO (or more) + # ADM secondaries. Resolve these on which matched a primary target + # ADM (rather than just a source from the sweeps) and then ALSO on + # ADM which has highest priority (via True/False * PRIORITY_INIT). alldups = [] - # for _, dups in duplicates(primids): for _, dups in duplicates(primtargs['TARGETID']): - am = np.argmax(primtargs[dups]["PRIORITY_INIT"]) + am = np.argmax(primtargs[dups]["PRIM_MATCH"]*primtargs[dups]["PRIORITY_INIT"]) + dups = np.delete(dups, am) + alldups.append(dups) + # ADM catch cases where there are no duplicates. + if len(alldups) != 0: + alldups = np.hstack(alldups) + primtargs = np.delete(primtargs, alldups) + primids = np.delete(primids, alldups) + log.info("Discard {} cases where a primary matched multiple secondaries". + format(len(alldups))) + + # ADM matches could also have occurred ACROSS HEALPixels, producing + # ADM duplicated secondary targets with different TARGETIDs... + alldups = [] + for _, dups in duplicates(primids): + # ADM...resolve these on which matched a primary target (rather + # ADM than just a source from a sweeps files) and then ALSO on + # ADM which has the highest priority. The combination in the code + # ADM is, e.g., True/False (meaning 1/0) * PRIORITY_INIT. + am = np.argmax(primtargs[dups]["PRIM_MATCH"]*primtargs[dups]["PRIORITY_INIT"]) dups = np.delete(dups, am) alldups.append(dups) # ADM catch cases where there are no duplicates. @@ -398,7 +419,8 @@ def add_primary_info(scxtargs, priminfodir): alldups = np.hstack(alldups) primtargs = np.delete(primtargs, alldups) primids = np.delete(primids, alldups) - log.debug("Discarded {} primary duplicates".format(len(alldups))) + log.info("Remove {} other cases where a secondary matched several primaries". + format(len(alldups))) # ADM we already know that all primaries match a secondary, so, # ADM for speed, we can reduce to the matching set. @@ -605,6 +627,8 @@ def match_secondary(primtargs, scxdir, scndout, sep=1., log.info("Read {} sources from {}/{} sweep files...t={:.1f}s".format( np.sum(inhp), ifil+1, len(swfiles), time()-start)) swobjs = np.concatenate(swobjs) + # ADM resolve so there are no duplicates across the N/S boundary. + swobjs = resolve(swobjs) log.info("Total sources read: {}".format(len(swobjs))) # ADM continue if there are sources in the pixels of interest. diff --git a/py/desitarget/sv1/data/sv1_targetmask.yaml b/py/desitarget/sv1/data/sv1_targetmask.yaml index f32e4306f..b031a649f 100644 --- a/py/desitarget/sv1/data/sv1_targetmask.yaml +++ b/py/desitarget/sv1/data/sv1_targetmask.yaml @@ -150,7 +150,7 @@ sv1_scnd_mask: {obsconditions: DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18, filename: 'veto'}] - [UDG, 1, "See $SCND_DIR/UDG.txt", {obsconditions: DARK, filename: 'UDG'}] - [FIRST_MALS, 2, "See $SCND_DIR/FIRST_MALS.txt", {obsconditions: DARK, filename: 'FIRST_MALS'}] - - [WD_BINARIES, 3, "See $SCND_DIR/WD_BINARIES.txt", {obsconditions: DARK, filename: 'WD_BINARIES'}] + - [WD_BINARIES, 3, "See $SCND_DIR/WD_BINARIES.txt", {obsconditions: BRIGHT|DARK, filename: 'WD_BINARIES'}] - [LBG_TOMOG, 4, "See $SCND_DIR/LBG_TOMOG.txt", {obsconditions: DARK, filename: 'LBG_TOMOG'}] - [QSO_RED, 5, "See $SCND_DIR/QSO_RED.ipynb", {obsconditions: DARK, filename: 'QSO_RED'}] - [M31_KNOWN, 6, "See $SCND_DIR/M31_KNOWN.txt", {obsconditions: DARK, filename: 'M31_KNOWN'}] @@ -160,8 +160,8 @@ sv1_scnd_mask: - [MWS_CLUS_GAL_DEEP, 10, "See $SCND_DIR/MWS_CLUS_GAL_DEEP.txt", {obsconditions: DARK, filename: 'MWS_CLUS_GAL_DEEP'}] - [LOW_MASS_AGN, 11, "See $SCND_DIR/LOW_MASS_AGN.txt", {obsconditions: DARK, filename: 'LOW_MASS_AGN'}] - [FAINT_HPM, 12, "See $SCND_DIR/FAINT_HPM.txt", {obsconditions: DARK, filename: 'FAINT_HPM'}] - - [GW190412, 13, "See $SCND_DIR/GW190412.ipynb", {obsconditions: DARK|BRIGHT, filename: 'GW190412'}] - - [IC134191, 14, "See $SCND_DIR/IC134191.ipynb", {obsconditions: DARK|BRIGHT, filename: 'IC134191'}] + - [GW190412, 13, "See $SCND_DIR/GW190412.ipynb", {obsconditions: BRIGHT|DARK, filename: 'GW190412'}] + - [IC134191, 14, "See $SCND_DIR/IC134191.ipynb", {obsconditions: BRIGHT|DARK, filename: 'IC134191'}] - [PV_BRIGHT, 15, "See $SCND_DIR/PV_BRIGHT.ipynb", {obsconditions: BRIGHT, filename: 'PV_BRIGHT'}] - [PV_DARK, 16, "See $SCND_DIR/PV_DARK.ipynb", {obsconditions: DARK, filename: 'PV_DARK'}] - [LOW_Z, 17, "See $SCND_DIR/LOW_Z.ipynb", {obsconditions: DARK, filename: 'LOW_Z'}] @@ -180,7 +180,7 @@ sv1_scnd_mask: - [GAL_CLUS_2ND, 30, "See $SCND_DIR/GAL_CLUS_2ND.txt", {obsconditions: DARK, filename: 'GAL_CLUS_2ND'}] - [GAL_CLUS_SAT, 31, "See $SCND_DIR/GAL_CLUS_SAT.txt", {obsconditions: DARK, filename: 'GAL_CLUS_SAT'}] - [HSC_HIZ_SNE, 32, "See $SCND_DIR/HSC_HIZ_SNE.txt", {obsconditions: DARK, filename: 'HSC_HIZ_SNE'}] -# - [ISM_CGM_QGP, 33, "See $SCND_DIR", {obsconditions: DARK, filename: 'ISM_CGM_QGP'}] + - [ISM_CGM_QGP, 33, "See $SCND_DIR/ISM_CGM_QGP.txt", {obsconditions: DARK, filename: 'ISM_CGM_QGP'}] - [STRONG_LENS, 34, "See $SCND_DIR/STRONG_LENS.txt", {obsconditions: DARK, filename: 'STRONG_LENS'}] - [WISE_VAR_QSO, 35, "See $SCND_DIR/WISE_VAR_QSO.txt", {obsconditions: DARK, filename: 'WISE_VAR_QSO'}] - [MWS_CALIB, 36, "Stars with APOGEE/BOSS/GALAH/GAIAESO spectra", @@ -347,10 +347,10 @@ priorities: M31_QSO: SAME_AS_LBG_TOMOG M31_STAR: SAME_AS_LBG_TOMOG # MWS_DDOGIANTS: {UNOBS: 1450, DONE: 2, OBS: 1, DONOTOBSERVE: 0} - MWS_CLUS_GAL_DEEP: {UNOBS: 1450, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + MWS_CLUS_GAL_DEEP: {UNOBS: 4000, DONE: 2, OBS: 1, DONOTOBSERVE: 0} LOW_MASS_AGN: {UNOBS: 1025, DONE: 2, OBS: 1, DONOTOBSERVE: 0} FAINT_HPM: SAME_AS_LOW_MASS_AGN - GW190412: {UNOBS: 4000, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + GW190412: {UNOBS: 5000, DONE: 2, OBS: 1, DONOTOBSERVE: 0} IC134191: SAME_AS_GW190412 PV_BRIGHT: {UNOBS: 100, DONE: 2, OBS: 1, DONOTOBSERVE: 0} PV_DARK: {UNOBS: 100, DONE: 2, OBS: 1, DONOTOBSERVE: 0} @@ -370,7 +370,7 @@ priorities: GAL_CLUS_2ND: {UNOBS: 1024, DONE: 2, OBS: 1, DONOTOBSERVE: 0} GAL_CLUS_SAT: {UNOBS: 200, DONE: 2, OBS: 1, DONOTOBSERVE: 0} HSC_HIZ_SNE: SAME_AS_LBG_TOMOG -# ISM_CGM_QGP: {UNOBS: 4000, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + ISM_CGM_QGP: {UNOBS: 4000, DONE: 2, OBS: 1, DONOTOBSERVE: 0} STRONG_LENS: SAME_AS_UDG WISE_VAR_QSO: SAME_AS_QSO_RED MWS_CALIB: {UNOBS: 2995, DONE: 2, OBS: 1, DONOTOBSERVE: 0} @@ -527,7 +527,7 @@ numobs: GAL_CLUS_2ND: 100 GAL_CLUS_SAT: 100 HSC_HIZ_SNE: 100 -# ISM_CGM_QGP: 100 + ISM_CGM_QGP: 100 STRONG_LENS: 100 WISE_VAR_QSO: 100 MWS_CALIB: 100 diff --git a/py/desitarget/targets.py b/py/desitarget/targets.py index 6d05a6dd7..4c31b1871 100644 --- a/py/desitarget/targets.py +++ b/py/desitarget/targets.py @@ -819,7 +819,7 @@ def resolve(targets): ---------- targets : :class:`~numpy.ndarray` Rec array of targets. Must have columns "RA" and "DEC" and - either "RELEASE" or "PHOTSYS". + either "RELEASE" or "PHOTSYS" or "TARGETID". Returns ------- @@ -833,7 +833,11 @@ def resolve(targets): if 'PHOTSYS' in targets.dtype.names: photsys = targets["PHOTSYS"] else: - photsys = release_to_photsys(targets["RELEASE"]) + if 'RELEASE' in targets.dtype.names: + photsys = release_to_photsys(targets["RELEASE"]) + else: + _, _, release, _, _, _ = decode_targetid(targets["TARGETID"]) + photsys = release_to_photsys(release) # ADM a flag of which targets are from the 'N' photometry. from desitarget.cuts import _isonnorthphotsys From 34a3a10302bb39e883490e198dde38875f9e8246 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Mon, 4 Jan 2021 17:21:11 -0800 Subject: [PATCH 13/30] minor code style --- py/desitarget/test/test_units.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/desitarget/test/test_units.py b/py/desitarget/test/test_units.py index 208224b92..a8a7ab0b1 100644 --- a/py/desitarget/test/test_units.py +++ b/py/desitarget/test/test_units.py @@ -54,7 +54,7 @@ def test_fits_units(self): self.assertEqual(list(uniq), parsed) def test_quantities(self): - """Test all data model quantities in are in the units yaml file. + """Test all data model quantities are in the units yaml file. """ missing = [dmn for dmn in self.dmnames if dmn not in self.units] msg = 'These quantities are missing in {}'.format(self.fn) From 125ac82b0f9dff10d5292d235cda862464c3c141 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Mon, 4 Jan 2021 22:35:05 -0800 Subject: [PATCH 14/30] update the changes file --- doc/changes.rst | 6 ++++++ py/desitarget/secondary.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/changes.rst b/doc/changes.rst index 63bd659ab..ead8de925 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -5,6 +5,11 @@ desitarget Change Log 0.47.1 (unreleased) ------------------- +* First run of secondaries with real target files [`PR #669`_]. Includes: + * General optimization, updating and debugging of the secondary code. + * Get `TARGETIDs` from the input sweeps, not just the actual targets. + * Add the first full bitmask for the SV1 secondary target files. + * Updates to the data model to better reflect the primary targets. * Updates and bug fixes for DR9 now SV is on-sky [`PR #665`_]. Includes: * Pass `MASKBITS` column forward for GFAs. * Bug fixes necessitated by target files having a second extension. @@ -19,6 +24,7 @@ desitarget Change Log .. _`PR #636`: https://github.com/desihub/desitarget/pull/636 .. _`PR #664`: https://github.com/desihub/desitarget/pull/664 .. _`PR #665`: https://github.com/desihub/desitarget/pull/665 +.. _`PR #669`: https://github.com/desihub/desitarget/pull/669 0.47.0 (2020-12-10) ------------------- diff --git a/py/desitarget/secondary.py b/py/desitarget/secondary.py index 3085522e9..707dbaa27 100644 --- a/py/desitarget/secondary.py +++ b/py/desitarget/secondary.py @@ -518,7 +518,7 @@ def match_secondary(primtargs, scxdir, scndout, sep=1., raise ValueError(msg) # ADM warn the user if the secondary and primary samples are "large". - big = 500000 + big = 1e6 if np.sum(inhp) > big and len(primtargs) > big: log.warning('Large secondary (N={}) and primary (N={}) samples' .format(np.sum(inhp), len(primtargs))) From 8ea3774ac5438f1da4fa39c89813ff29a49d5475 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 5 Jan 2021 14:12:24 -0800 Subject: [PATCH 15/30] commenting out the PV targeting bits, for now, as there may be a bug in their files; can easily add these targets back in later --- py/desitarget/sv1/data/sv1_targetmask.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/py/desitarget/sv1/data/sv1_targetmask.yaml b/py/desitarget/sv1/data/sv1_targetmask.yaml index b031a649f..d6504489a 100644 --- a/py/desitarget/sv1/data/sv1_targetmask.yaml +++ b/py/desitarget/sv1/data/sv1_targetmask.yaml @@ -162,8 +162,8 @@ sv1_scnd_mask: - [FAINT_HPM, 12, "See $SCND_DIR/FAINT_HPM.txt", {obsconditions: DARK, filename: 'FAINT_HPM'}] - [GW190412, 13, "See $SCND_DIR/GW190412.ipynb", {obsconditions: BRIGHT|DARK, filename: 'GW190412'}] - [IC134191, 14, "See $SCND_DIR/IC134191.ipynb", {obsconditions: BRIGHT|DARK, filename: 'IC134191'}] - - [PV_BRIGHT, 15, "See $SCND_DIR/PV_BRIGHT.ipynb", {obsconditions: BRIGHT, filename: 'PV_BRIGHT'}] - - [PV_DARK, 16, "See $SCND_DIR/PV_DARK.ipynb", {obsconditions: DARK, filename: 'PV_DARK'}] +# - [PV_BRIGHT, 15, "See $SCND_DIR/PV_BRIGHT.ipynb", {obsconditions: BRIGHT, filename: 'PV_BRIGHT'}] +# - [PV_DARK, 16, "See $SCND_DIR/PV_DARK.ipynb", {obsconditions: DARK, filename: 'PV_DARK'}] - [LOW_Z, 17, "See $SCND_DIR/LOW_Z.ipynb", {obsconditions: DARK, filename: 'LOW_Z'}] - [BHB, 18, "See $SCND_DIR/BHB.txt", {obsconditions: DARK, filename: 'BHB'}] - [SPCV, 19, "See $SCND_DIR/SPCV.txt", {obsconditions: DARK, filename: 'SPCV'}] @@ -352,8 +352,8 @@ priorities: FAINT_HPM: SAME_AS_LOW_MASS_AGN GW190412: {UNOBS: 5000, DONE: 2, OBS: 1, DONOTOBSERVE: 0} IC134191: SAME_AS_GW190412 - PV_BRIGHT: {UNOBS: 100, DONE: 2, OBS: 1, DONOTOBSERVE: 0} - PV_DARK: {UNOBS: 100, DONE: 2, OBS: 1, DONOTOBSERVE: 0} +# PV_BRIGHT: {UNOBS: 100, DONE: 2, OBS: 1, DONOTOBSERVE: 0} +# PV_DARK: {UNOBS: 100, DONE: 2, OBS: 1, DONOTOBSERVE: 0} LOW_Z: {UNOBS: 90, DONE: 2, OBS: 1, DONOTOBSERVE: 0} BHB: {UNOBS: 1950, DONE: 2, OBS: 1, DONOTOBSERVE: 0} SPCV: SAME_AS_LOW_MASS_AGN @@ -509,8 +509,8 @@ numobs: FAINT_HPM: 100 GW190412: 100 IC134191: 100 - PV_BRIGHT: 100 - PV_DARK: 100 +# PV_BRIGHT: 100 +# PV_DARK: 100 LOW_Z: 100 BHB: 100 SPCV: 100 From 0aceb2afe478c1d15f2eca8f1f420ad3896f6932 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Tue, 5 Jan 2021 22:23:46 -0800 Subject: [PATCH 16/30] restore PV targeting bits, bug is fixed; split HPM class into BRIGHT And FAINT for BRIGHT and DARK obsconditions --- py/desitarget/sv1/data/sv1_targetmask.yaml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/py/desitarget/sv1/data/sv1_targetmask.yaml b/py/desitarget/sv1/data/sv1_targetmask.yaml index d6504489a..c0b599815 100644 --- a/py/desitarget/sv1/data/sv1_targetmask.yaml +++ b/py/desitarget/sv1/data/sv1_targetmask.yaml @@ -162,8 +162,8 @@ sv1_scnd_mask: - [FAINT_HPM, 12, "See $SCND_DIR/FAINT_HPM.txt", {obsconditions: DARK, filename: 'FAINT_HPM'}] - [GW190412, 13, "See $SCND_DIR/GW190412.ipynb", {obsconditions: BRIGHT|DARK, filename: 'GW190412'}] - [IC134191, 14, "See $SCND_DIR/IC134191.ipynb", {obsconditions: BRIGHT|DARK, filename: 'IC134191'}] -# - [PV_BRIGHT, 15, "See $SCND_DIR/PV_BRIGHT.ipynb", {obsconditions: BRIGHT, filename: 'PV_BRIGHT'}] -# - [PV_DARK, 16, "See $SCND_DIR/PV_DARK.ipynb", {obsconditions: DARK, filename: 'PV_DARK'}] + - [PV_BRIGHT, 15, "See $SCND_DIR/PV_BRIGHT.ipynb", {obsconditions: BRIGHT, filename: 'PV_BRIGHT'}] + - [PV_DARK, 16, "See $SCND_DIR/PV_DARK.ipynb", {obsconditions: DARK, filename: 'PV_DARK'}] - [LOW_Z, 17, "See $SCND_DIR/LOW_Z.ipynb", {obsconditions: DARK, filename: 'LOW_Z'}] - [BHB, 18, "See $SCND_DIR/BHB.txt", {obsconditions: DARK, filename: 'BHB'}] - [SPCV, 19, "See $SCND_DIR/SPCV.txt", {obsconditions: DARK, filename: 'SPCV'}] @@ -191,6 +191,7 @@ sv1_scnd_mask: {obsconditions: DARK|GRAY|BRIGHT, filename: 'MWS_MAIN_CLUSTER_SV'}] - [MWS_RRLYR, 39, "Main survey halo tracers (RR Lyrae)", {obsconditions: DARK|GRAY|BRIGHT, filename: 'MWS_RRLYR'}] + - [BRIGHT_HPM, 40, "See $SCND_DIR/BRIGHT_HPM.txt", {obsconditions: BRIGHT, filename: 'BRIGHT_HPM'}] #- Observation State #- if a target passes more than one target bit, it is possible that one bit @@ -352,8 +353,8 @@ priorities: FAINT_HPM: SAME_AS_LOW_MASS_AGN GW190412: {UNOBS: 5000, DONE: 2, OBS: 1, DONOTOBSERVE: 0} IC134191: SAME_AS_GW190412 -# PV_BRIGHT: {UNOBS: 100, DONE: 2, OBS: 1, DONOTOBSERVE: 0} -# PV_DARK: {UNOBS: 100, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + PV_BRIGHT: {UNOBS: 100, DONE: 2, OBS: 1, DONOTOBSERVE: 0} + PV_DARK: {UNOBS: 100, DONE: 2, OBS: 1, DONOTOBSERVE: 0} LOW_Z: {UNOBS: 90, DONE: 2, OBS: 1, DONOTOBSERVE: 0} BHB: {UNOBS: 1950, DONE: 2, OBS: 1, DONOTOBSERVE: 0} SPCV: SAME_AS_LOW_MASS_AGN @@ -377,6 +378,7 @@ priorities: BACKUP_CALIB: SAME_AS_MWS_CALIB MWS_MAIN_CLUSTER_SV: {UNOBS: 1450, DONE: 400, OBS: 1, DONOTOBSERVE: 0} MWS_RRLYR: {UNOBS: 1450, DONE: 400, OBS: 1, DONOTOBSERVE: 0} + BRIGHT_HPM: SAME_AS_LOW_MASS_AGN # ADM INITIAL number of observations (NUMOBS) for each target bit # ADM SAME_AS_XXX means to use the NUMOBS for bitname XXX @@ -509,8 +511,8 @@ numobs: FAINT_HPM: 100 GW190412: 100 IC134191: 100 -# PV_BRIGHT: 100 -# PV_DARK: 100 + PV_BRIGHT: 100 + PV_DARK: 100 LOW_Z: 100 BHB: 100 SPCV: 100 @@ -534,3 +536,4 @@ numobs: MWS_MAIN_CLUSTER_SV: 1 MWS_RRLYR: 100 BACKUP_CALIB: SAME_AS_MWS_CALIB + BRIGHT_HPM: 100 \ No newline at end of file From ff06f6a8e5bd43a42685f32bda373d1dab007c32 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Wed, 6 Jan 2021 16:32:00 -0800 Subject: [PATCH 17/30] add columns needed by fiberassign --- py/desitarget/secondary.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/py/desitarget/secondary.py b/py/desitarget/secondary.py index 707dbaa27..1bb353557 100644 --- a/py/desitarget/secondary.py +++ b/py/desitarget/secondary.py @@ -55,6 +55,7 @@ from desitarget.internal import sharedmem from desitarget.geomask import radec_match_to, add_hp_neighbors, is_in_hp +from desitarget.gaiamatch import gaiadatamodel from desitarget.targets import encode_targetid, main_cmx_or_sv, resolve from desitarget.targets import set_obsconditions, initial_priority_numobs @@ -81,13 +82,17 @@ # or svX/data/svX_targetmask.yaml (scnd_mask). # ADM Note that TARGETID for secondary-only targets is unique because # ADM RELEASE is < 1000 (before DR1) for secondary-only targets. +# ADM also add needed columns for fiberassign from the Gaia data model. +gaiacols = ["PARALLAX", "GAIA_PHOT_G_MEAN_MAG", 'GAIA_ASTROMETRIC_EXCESS_NOISE'] +gaiadt = [(gaiadatamodel[gaiacols].dtype.names[i], + gaiadatamodel[gaiacols].dtype[i].str) for i in range(len(gaiacols))] + outdatamodel = np.array([], dtype=[ ('RA', '>f8'), ('DEC', '>f8'), ('PMRA', '>f4'), ('PMDEC', '>f4'), - ('REF_EPOCH', '>f4'), ('OVERRIDE', '?'), + ('REF_EPOCH', '>f4'), ('OVERRIDE', '?')] + gaiadt + [ ('TARGETID', '>i8'), ('DESI_TARGET', '>i8'), ('SCND_TARGET', '>i8'), ('PRIORITY_INIT', '>i8'), ('SUBPRIORITY', '>f8'), - ('NUMOBS_INIT', '>i8'), ('OBSCONDITIONS', '>i8') -]) + ('NUMOBS_INIT', '>i8'), ('OBSCONDITIONS', '>i8')]) # ADM extra columns that are used during processing but are # ADM not an official part of the input or output data model. From 8b925b874e3cba29567723c9d417e8230429b75a Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Thu, 7 Jan 2021 14:44:46 -0800 Subject: [PATCH 18/30] allow secondary targets to be read by the desitarget.io.read_target* routines --- py/desitarget/io.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/py/desitarget/io.py b/py/desitarget/io.py index d7a7dfece..f92372536 100644 --- a/py/desitarget/io.py +++ b/py/desitarget/io.py @@ -2409,7 +2409,8 @@ def read_target_files(filename, columns=None, rows=None, header=False, """ start = time() # ADM start with some checking that this is a target file. - targtypes = "TARGETS", "GFA_TARGETS", "SKY_TARGETS", "MASKS", "MTL" + targtypes = ["TARGETS", "GFA_TARGETS", "SKY_TARGETS", + "MASKS", "MTL", "SCND_TARGETS"] # ADM read in the FITS extension info. f = fitsio.FITS(filename) if len(f) != 2: From d170cd2b6180f59bc759f8bfe572d58b679c5151 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 8 Jan 2021 10:29:29 -0800 Subject: [PATCH 19/30] add a check that the input secondary data have reasonable coordinates --- py/desitarget/secondary.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/py/desitarget/secondary.py b/py/desitarget/secondary.py index 1bb353557..a35cdba16 100644 --- a/py/desitarget/secondary.py +++ b/py/desitarget/secondary.py @@ -318,11 +318,19 @@ def read_files(scxdir, scnd_mask): scxin = np.atleast_1d(scxin) # ADM assert the data model. - msg = "Data model doesn't match {} in {}.fits".format( - indatamodel.dtype, fn) + msg = "Data model doesn't match {} in {}".format(indatamodel.dtype, fn) for col in indatamodel.dtype.names: assert scxin[col].dtype == indatamodel[col].dtype, msg + # ADM check RA/Dec are reasonable. + outofbounds = ((scxin["RA"] >= 360.) | (scxin["RA"] < 0) | + (scxin["DEC"] > 90) | (scxin["DEC"] < -90)) + if np.any(outofbounds): + msg = "RA/Dec outside of range in {}; RA={}, Dec={}".format( + fn, scxin["RA"][outofbounds], scxin["DEC"][outofbounds]) + log.error(msg) + raise IOError(msg) + # ADM the default is 2015.5 for the REF_EPOCH. ii = scxin["REF_EPOCH"] == 0 scxin["REF_EPOCH"][ii] = 2015.5 From d3dbe9c5544ea995d68a113537cac59b9768a534 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 8 Jan 2021 10:58:35 -0800 Subject: [PATCH 20/30] add Gaia-only standard star bits to the MWS_MASK --- py/desitarget/data/targetmask.yaml | 13 +++++++++++++ py/desitarget/sv1/data/sv1_targetmask.yaml | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/py/desitarget/data/targetmask.yaml b/py/desitarget/data/targetmask.yaml index 21d19a02f..61b5daee6 100644 --- a/py/desitarget/data/targetmask.yaml +++ b/py/desitarget/data/targetmask.yaml @@ -101,6 +101,11 @@ mws_mask: - [MWS_MAIN_RED_NORTH, 12, "MWS magnitude limited red sample tuned for Bok/Mosaic", {obsconditions: BRIGHT}] - [MWS_MAIN_RED_SOUTH, 13, "MWS magnitude limited red sample tuned for DECam", {obsconditions: BRIGHT}] + # ADM Standard stars based only on Gaia. + - [GAIA_STD_FAINT, 33, "Standard stars for dark/gray conditions", {obsconditions: DARK|GRAY}] + - [GAIA_STD_WD, 34, "White Dwarf stars", {obsconditions: DARK|GRAY|BRIGHT}] + - [GAIA_STD_BRIGHT, 35, "Standard stars for BRIGHT conditions", {obsconditions: BRIGHT}] + # ADM back-up targets for poor conditions and as filler. - [BACKUP_BRIGHT, 60, "Bright backup Gaia targets", {obsconditions: DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18}] - [BACKUP_FAINT, 61, "Fainter backup Gaia targets", {obsconditions: DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18}] @@ -231,6 +236,10 @@ priorities: BACKUP_BRIGHT: {UNOBS: 9, DONE: 9, OBS: 9, DONOTOBSERVE: 0} BACKUP_FAINT: {UNOBS: 8, DONE: 8, OBS: 8, DONOTOBSERVE: 0} BACKUP_VERY_FAINT: {UNOBS: 7, DONE: 7, OBS: 7, DONOTOBSERVE: 0} + # ADM Standards are special; priorities don't apply. + GAIA_STD_FAINT: -1 + GAIA_STD_WD: -1 + GAIA_STD_BRIGHT: -1 # ADM secondary target priorities. Probably all have very low UNOBS... scnd_mask: @@ -305,6 +314,10 @@ numobs: BACKUP_BRIGHT: 1 BACKUP_FAINT: SAME_AS_BACKUP_BRIGHT BACKUP_VERY_FAINT: SAME_AS_BACKUP_BRIGHT + # ADM Standards are special; numobs doesn't apply. + GAIA_STD_FAINT: -1 + GAIA_STD_WD: -1 + GAIA_STD_BRIGHT: -1 # ADM initial number of observations for secondary targets scnd_mask: diff --git a/py/desitarget/sv1/data/sv1_targetmask.yaml b/py/desitarget/sv1/data/sv1_targetmask.yaml index c0b599815..36d5b3cf2 100644 --- a/py/desitarget/sv1/data/sv1_targetmask.yaml +++ b/py/desitarget/sv1/data/sv1_targetmask.yaml @@ -136,6 +136,11 @@ sv1_mws_mask: - [MWS_MAIN_FAINT_NORTH, 15, "MWS magnitude limited sample from DECam", {obsconditions: BRIGHT|GRAY|DARK}] - [MWS_MAIN_FAINT_SOUTH, 16, "MWS magnitude limited sample from DECam", {obsconditions: BRIGHT|GRAY|DARK}] + # ADM Standard stars based only on Gaia. + - [GAIA_STD_FAINT, 33, "Standard stars for dark/gray conditions", {obsconditions: DARK|GRAY}] + - [GAIA_STD_WD, 34, "White Dwarf stars", {obsconditions: DARK|GRAY|BRIGHT}] + - [GAIA_STD_BRIGHT, 35, "Standard stars for BRIGHT conditions", {obsconditions: BRIGHT}] + # ADM back-up targets for poor conditions. - [BACKUP_BRIGHT, 60, "Bright backup Gaia targets", {obsconditions: DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18}] - [BACKUP_FAINT, 61, "Fainter backup Gaia targets", {obsconditions: DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18}] @@ -335,6 +340,10 @@ priorities: BACKUP_BRIGHT: {UNOBS: 9, DONE: 9, OBS: 9, DONOTOBSERVE: 0} BACKUP_FAINT: {UNOBS: 8, DONE: 8, OBS: 8, DONOTOBSERVE: 0} BACKUP_VERY_FAINT: {UNOBS: 7, DONE: 7, OBS: 7, DONOTOBSERVE: 0} + # ADM Standards are special; priorities don't apply. + GAIA_STD_FAINT: -1 + GAIA_STD_WD: -1 + GAIA_STD_BRIGHT: -1 # ADM secondary target priorities. Probably all have very low UNOBS sv1_scnd_mask: @@ -493,6 +502,10 @@ numobs: BACKUP_BRIGHT: 1 BACKUP_FAINT: SAME_AS_BACKUP_BRIGHT BACKUP_VERY_FAINT: SAME_AS_BACKUP_BRIGHT + # ADM Standards are special; numobs doesn't apply. + GAIA_STD_FAINT: -1 + GAIA_STD_WD: -1 + GAIA_STD_BRIGHT: -1 # ADM initial number of observations for secondary targets sv1_scnd_mask: From f7a5b0e5e1de86c76ddb669ef156eb85cd14a634 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 8 Jan 2021 15:52:43 -0800 Subject: [PATCH 21/30] add function to create new Gaia-only standards. Also modify preparation of data to work with Gaia-only files --- py/desitarget/cuts.py | 66 ++++++++++++------ py/desitarget/sv1/sv1_cuts.py | 125 +++++++++++++++++++++++++++++++++- 2 files changed, 168 insertions(+), 23 deletions(-) diff --git a/py/desitarget/cuts.py b/py/desitarget/cuts.py index 61805125a..15e8dca0b 100644 --- a/py/desitarget/cuts.py +++ b/py/desitarget/cuts.py @@ -1688,7 +1688,7 @@ def _prepare_optical_wise(objects, mask=True): dchisq, deltaChi2, maskbits, refcat) -def _prepare_gaia(objects, colnames=None): +def _prepare_gaia(objects, colnames=None, gaiaonly=False): """Process the various Gaia inputs for target selection.""" if colnames is None: @@ -1698,14 +1698,21 @@ def _prepare_gaia(objects, colnames=None): # ADM if we don't have REF_CAT in the sweeps use the # ADM minimum value of REF_ID to identify Gaia sources. This will # ADM introduce a small number (< 0.001%) of Tycho-only sources. - gaia = objects['REF_ID'] > 0 - if "REF_CAT" in colnames: - gaia = (objects['REF_CAT'] == b'G2') | (objects['REF_CAT'] == 'G2') + if gaiaonly: + gaia = np.ones_like(objects, dtype=bool) + else: + gaia = objects['REF_ID'] > 0 + if "REF_CAT" in colnames: + gaia = (objects['REF_CAT'] == b'G2') | (objects['REF_CAT'] == 'G2') pmra = objects['PMRA'] pmdec = objects['PMDEC'] - pmraivar = objects['PMRA_IVAR'] parallax = objects['PARALLAX'] - parallaxivar = objects['PARALLAX_IVAR'] + if gaiaonly: + pmraivar = 1./objects['PMRA_ERROR']**2. + parallaxivar = 1./objects['PARALLAX_ERROR']**2. + else: + pmraivar = objects['PMRA_IVAR'] + parallaxivar = objects['PARALLAX_IVAR'] # ADM derive the parallax/parallax_error, but set to 0 where the error is bad parallaxovererror = np.where(parallaxivar > 0., parallax*np.sqrt(parallaxivar), 0.) @@ -1714,27 +1721,35 @@ def _prepare_gaia(objects, colnames=None): notzero = parallaxivar > 0 if np.sum(notzero) > 0: parallaxerr[notzero] = 1 / np.sqrt(parallaxivar[notzero]) - gaiagmag = objects['GAIA_PHOT_G_MEAN_MAG'] - gaiabmag = objects['GAIA_PHOT_BP_MEAN_MAG'] - gaiarmag = objects['GAIA_PHOT_RP_MEAN_MAG'] - gaiaaen = objects['GAIA_ASTROMETRIC_EXCESS_NOISE'] + + prefix = "GAIA" + if gaiaonly: + prefix = "" + gaiagmag = objects[prefix + "PHOT_G_MEAN_MAG"] + gaiabmag = objects[prefix + "PHOT_BP_MEAN_MAG"] + gaiarmag = objects[prefix + "PHOT_RP_MEAN_MAG"] + gaiaaen = objects[prefix + "ASTROMETRIC_EXCESS_NOISE"] # ADM a mild hack, as GAIA_DUPLICATED_SOURCE was a 0/1 integer at some point. - gaiadupsource = objects['GAIA_DUPLICATED_SOURCE'] + gaiadupsource = objects[prefix + "DUPLICATED_SOURCE"] if issubclass(gaiadupsource.dtype.type, np.integer): if len(set(np.atleast_1d(gaiadupsource)) - set([0, 1])) == 0: - gaiadupsource = objects['GAIA_DUPLICATED_SOURCE'].astype(bool) + gaiadupsource = objects[prefix + "DUPLICATED_SOURCE"].astype(bool) # For BGS target selection. # ADM first guard against FLUX_R < 0 (I've checked this generates # ADM the same set of targets as Grr = NaN). - Grr = gaiagmag - 22.5 + 2.5*np.log10(1e-16) - ii = objects['FLUX_R'] > 0 - # ADM catch the case where Grr is a scalar. - if isinstance(Grr, np.float): - if ii: - Grr = gaiagmag - 22.5 + 2.5*np.log10(objects['FLUX_R']) + if gaiaonly: + Grr = np.zeros_like(objects[prefix + "PHOT_G_MEAN_MAG"]) + Grr = np.nan else: - Grr[ii] = gaiagmag[ii] - 22.5 + 2.5*np.log10(objects['FLUX_R'][ii]) + Grr = gaiagmag - 22.5 + 2.5*np.log10(1e-16) + ii = objects['FLUX_R'] > 0 + # ADM catch the case where Grr is a scalar. + if isinstance(Grr, np.float): + if ii: + Grr = gaiagmag - 22.5 + 2.5*np.log10(objects['FLUX_R']) + else: + Grr[ii] = gaiagmag[ii] - 22.5 + 2.5*np.log10(objects['FLUX_R'][ii]) # ADM If proper motion is not NaN, 31 parameters were solved for # ADM in Gaia astrometry. Or, gaiaparamssolved should be 3 for NaNs). @@ -2238,18 +2253,27 @@ def apply_cuts_gaia(numproc=4, survey='main', nside=None, pixlist=None): # ADM field which we need to remove before finalizing. gaiaobjs = rfn.drop_fields(gaiaobjs, "TARGETID") - primary = np.ones_like(gaiaobjs, dtype=bool) - # ADM the relevant input quantities. ra = gaiaobjs["RA"] dec = gaiaobjs["DEC"] gaiagmag = gaiaobjs["GAIA_PHOT_G_MEAN_MAG"] # ADM determine if an object is a BACKUP target. + primary = np.ones_like(gaiaobjs, dtype=bool) backup_bright, backup_faint, backup_very_faint = targcuts.isBACKUP( ra=ra, dec=dec, gaiagmag=gaiagmag, primary=primary ) + # ADM determine if a target is a Gaia-only standard. + primary = np.ones_like(gaiaobjs, dtype=bool) + std_faint, std_bright, std_wd = targcuts.isGAIA_STD( + ra=ra, dec=dec, galb=galb, gaiaaen=gaiaaen, pmra=pmra, pmdec=pmdec, + parallax=parallax, parallaxovererror=parallaxovererror, + gaiabprpfactor=gaiabprpfactor, gaiasigma5dmax=gaiasigma5dmax, + gaiagmag=gaiagmag, gaiabmag=gaiabmag, gaiarmag=gaiarmag, + gaiadupsource=gaiadupsource, gaiaparamssolved=gaiaparamssolved, + primary=primary) + # ADM Construct the target flag bits. mws_target = backup_bright * mws_mask.BACKUP_BRIGHT mws_target |= backup_faint * mws_mask.BACKUP_FAINT diff --git a/py/desitarget/sv1/sv1_cuts.py b/py/desitarget/sv1/sv1_cuts.py index c3a622a1a..39523bb00 100644 --- a/py/desitarget/sv1/sv1_cuts.py +++ b/py/desitarget/sv1/sv1_cuts.py @@ -31,6 +31,127 @@ start = time() +def isGAIA_STD(ra=None, dec=None, galb=None, gaiaaen=None, pmra=None, pmdec=None, + parallax=None, parallaxovererror=None, gaiabprpfactor=None, + gaiasigma5dmax=None, gaiagmag=None, gaiabmag=None, gaiarmag=None, + gaiadupsource=None, gaiaparamssolved=None, + primary=None, test=False): + """Standards based solely on Gaia data. + + Parameters + ---------- + test : :class:`bool`, optional, defaults to ``False`` + If ``True``, then we're running unit tests and don't have to + find and read every possible Gaia file. + + see :func:`~desitarget.sv1.sv1_cuts.set_target_bits` for parameters. + + Returns + ------- + :class:`array_like` + ``True`` if the object is a bright "GAIA_STD_FAINT" target. + :class:`array_like` + ``True`` if the object is a faint "GAIA_STD_BRIGHT" target. + :class:`array_like` + ``True`` if the object is a white dwarf "GAIA_STD_WD" target. + + Notes + ----- + - Current version (01/08/21) is version XXX on `the SV wiki`_. + """ + if primary is None: + primary = np.ones_like(gaiagmag, dtype='?') + + # ADM restrict all classes to dec >= -30. + primary &= dec >= -30. + std = primary.copy() + + # ADM the regular "standards" codes need to know whether something has + # ADM a Gaia match. Here, everything is a Gaia match. + gaia = np.ones_like(gaiagmag, dtype='?') + + # ADM determine the Gaia-based white dwarf standards. + std_wd = isMWS_WD( + primary=primary, gaia=gaia, galb=galb, astrometricexcessnoise=gaiaaen, + pmra=pmra, pmdec=pmdec, parallax=parallax, + parallaxovererror=parallaxovererror, photbprpexcessfactor=gaiabprpfactor, + astrometricsigma5dmax=gaiasigma5dmax, gaiagmag=gaiagmag, + gaiabmag=gaiabmag, gaiarmag=gaiarmag + ) + + # ADM apply the Gaia quality cuts for standards. + std &= isSTD_gaia(primary=primary, gaia=gaia, astrometricexcessnoise=gaiaaen, + pmra=pmra, pmdec=pmdec, parallax=parallax, + dupsource=gaiadupsource, paramssolved=gaiaparamssolved, + gaiagmag=gaiagmag, gaiabmag=gaiabmag, gaiarmag=gaiarmag) + + # ADM restrict to point sources + ispsf = np.logical_or( + (gaiagmag <= 19.) * (gaiaaen < 10.**0.5), + (gaiagmag >= 19.) * (gaiaaen < 10.**(0.5 + 0.2*(gaiagmag - 19.))) + ) + std &= ispsf + + # ADM apply the Gaia color cuts for standards. + bprp = gaiabmag - gaiarmag + gbp = gaiagmag - gaiabmag + std &= bprp > 0.2 + std &= bprp < 0.9 + std &= gbp > -1*bprp/2.0 + std &= gbp < 0.3-bprp/2.0 + + # ADM remove any sources that have neighbors in Gaia within 3.5"... + # ADM for speed, run only sources for which std is still True. + log.info("Isolating Gaia-only standards...t={:.1f}s".format(time()-start)) + ii_true = np.where(std)[0] + if len(ii_true) > 0: + # ADM determine the pixels of interest. + theta, phi = np.radians(90-dec), np.radians(ra) + pixlist = list(set(hp.ang2pix(nside, theta, phi, nest=True))) + # ADM read in the necessary Gaia files. + fns = find_gaia_files_hp(nside, pixlist, neighbors=True) + gaiaobjs = [] + gaiacols = ["RA", "DEC", "PHOT_G_MEAN_MAG", "PHOT_RP_MEAN_MAG"] + for i, fn in enumerate(fns): + if i % 25 == 0: + log.info("Read {}/{} files for STD_DITHER_GAIA...t={:.1f}s" + .format(i, len(fns), time()-start)) + try: + gaiaobjs.append(fitsio.read(fn, columns=gaiacols)) + except OSError: + if test: + pass + else: + msg = "failed to find or open the following file: (ffopen) " + msg += fn + log.critical(msg) + raise OSError + gaiaobjs = np.concatenate(gaiaobjs) + # ADM match the standards to the broader Gaia sources at 3.5". + matchrad = 3.5*u.arcsec + cstd = SkyCoord(ra[ii_true]*u.degree, dec[ii_true]*u.degree) + cgaia = SkyCoord(gaiaobjs["RA"]*u.degree, gaiaobjs["DEC"]*u.degree) + idstd, idgaia, d2d, _ = cgaia.search_around_sky(cstd, matchrad) + # ADM remove source matches with d2d=0 (i.e. the source itself!). + idgaia, idstd = idgaia[d2d > 0], idstd[d2d > 0] + # ADM remove matches within 5 mags of a Gaia source. + badmag = ( + (gmag[ii_true][idstd] + 5 > gaiaobjs["PHOT_G_MEAN_MAG"][idgaia]) | + (rmag[ii_true][idstd] + 5 > gaiaobjs["PHOT_RP_MEAN_MAG"][idgaia])) + std[ii_true[idstd][badmag]] = False + + # ADM add the brightness cuts in Gaia G-band. + std_bright = std.copy() + std_bright &= gaiagmag >= 15 + std_bright &= gaiagmag < 18 + + std_faint = std.copy() + std_faint &= gaiagmag >= 16 + std_faint &= gaiagmag < 19 + + return std_faint, std_bright, std_wd + + def isBACKUP(ra=None, dec=None, gaiagmag=None, primary=None): """BACKUP targets based on Gaia magnitudes. @@ -382,7 +503,7 @@ def isSTD_gaia(primary=None, gaia=None, astrometricexcessnoise=None, see :func:`~desitarget.sv1.sv1_cuts.isSTD` for other details. """ if primary is None: - primary = np.ones_like(gflux, dtype='?') + primary = np.ones_like(gaiagmag, dtype='?') std = primary.copy() # ADM Bp and Rp are both measured. @@ -1454,7 +1575,7 @@ def isMWS_WD(primary=None, gaia=None, galb=None, astrometricexcessnoise=None, Parameters ---------- - see :func:`~desitarget.sv1.sv1_cuts.set_target_bits` for other parameters. + see :func:`~desitarget.sv1.sv1_cuts.set_target_bits` for parameters. Returns ------- From ad0f2287a2e36e324909fa5888c4b162d327afc2 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 8 Jan 2021 16:10:58 -0800 Subject: [PATCH 22/30] set the bits for the Gaia-only standards in the MWS target column --- py/desitarget/cuts.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/py/desitarget/cuts.py b/py/desitarget/cuts.py index 15e8dca0b..5d00fbab9 100644 --- a/py/desitarget/cuts.py +++ b/py/desitarget/cuts.py @@ -1708,8 +1708,8 @@ def _prepare_gaia(objects, colnames=None, gaiaonly=False): pmdec = objects['PMDEC'] parallax = objects['PARALLAX'] if gaiaonly: - pmraivar = 1./objects['PMRA_ERROR']**2. - parallaxivar = 1./objects['PARALLAX_ERROR']**2. + pmraivar = 1./objects['PMRA_ERROR'].clip(1e-16)**2. + parallaxivar = 1./objects['PARALLAX_ERROR'].clip(1e-16)**2. else: pmraivar = objects['PMRA_IVAR'] parallaxivar = objects['PARALLAX_IVAR'] @@ -1722,7 +1722,7 @@ def _prepare_gaia(objects, colnames=None, gaiaonly=False): if np.sum(notzero) > 0: parallaxerr[notzero] = 1 / np.sqrt(parallaxivar[notzero]) - prefix = "GAIA" + prefix = "GAIA_" if gaiaonly: prefix = "" gaiagmag = objects[prefix + "PHOT_G_MEAN_MAG"] @@ -2256,7 +2256,9 @@ def apply_cuts_gaia(numproc=4, survey='main', nside=None, pixlist=None): # ADM the relevant input quantities. ra = gaiaobjs["RA"] dec = gaiaobjs["DEC"] - gaiagmag = gaiaobjs["GAIA_PHOT_G_MEAN_MAG"] + gaia, pmra, pmdec, parallax, parallaxovererror, parallaxerr, gaiagmag, gaiabmag, \ + gaiarmag, gaiaaen, gaiadupsource, Grr, gaiaparamssolved, gaiabprpfactor, \ + gaiasigma5dmax, galb = _prepare_gaia(objects, gaiaonly=True) # ADM determine if an object is a BACKUP target. primary = np.ones_like(gaiaobjs, dtype=bool) @@ -2278,6 +2280,9 @@ def apply_cuts_gaia(numproc=4, survey='main', nside=None, pixlist=None): mws_target = backup_bright * mws_mask.BACKUP_BRIGHT mws_target |= backup_faint * mws_mask.BACKUP_FAINT mws_target |= backup_very_faint * mws_mask.BACKUP_VERY_FAINT + mws_target |= std_faint * mws_mask.STD_FAINT + mws_target |= std_bright * mws_mask.STD_BRIGHT + mws_target |= std_wd * mws_mask.STD_WD bgs_target = np.zeros_like(mws_target) From 3bf60c64100339f8a03e5b5e1b764b2bad70acf2 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 8 Jan 2021 16:42:39 -0800 Subject: [PATCH 23/30] didn't actually need to modify the preparation of data to work with Gaia-only files --- py/desitarget/cuts.py | 54 ++++++++++++++++--------------------------- 1 file changed, 20 insertions(+), 34 deletions(-) diff --git a/py/desitarget/cuts.py b/py/desitarget/cuts.py index 5d00fbab9..028294687 100644 --- a/py/desitarget/cuts.py +++ b/py/desitarget/cuts.py @@ -1688,7 +1688,7 @@ def _prepare_optical_wise(objects, mask=True): dchisq, deltaChi2, maskbits, refcat) -def _prepare_gaia(objects, colnames=None, gaiaonly=False): +def _prepare_gaia(objects, colnames=None): """Process the various Gaia inputs for target selection.""" if colnames is None: @@ -1698,21 +1698,14 @@ def _prepare_gaia(objects, colnames=None, gaiaonly=False): # ADM if we don't have REF_CAT in the sweeps use the # ADM minimum value of REF_ID to identify Gaia sources. This will # ADM introduce a small number (< 0.001%) of Tycho-only sources. - if gaiaonly: - gaia = np.ones_like(objects, dtype=bool) - else: - gaia = objects['REF_ID'] > 0 - if "REF_CAT" in colnames: - gaia = (objects['REF_CAT'] == b'G2') | (objects['REF_CAT'] == 'G2') + gaia = objects['REF_ID'] > 0 + if "REF_CAT" in colnames: + gaia = (objects['REF_CAT'] == b'G2') | (objects['REF_CAT'] == 'G2') pmra = objects['PMRA'] pmdec = objects['PMDEC'] parallax = objects['PARALLAX'] - if gaiaonly: - pmraivar = 1./objects['PMRA_ERROR'].clip(1e-16)**2. - parallaxivar = 1./objects['PARALLAX_ERROR'].clip(1e-16)**2. - else: - pmraivar = objects['PMRA_IVAR'] - parallaxivar = objects['PARALLAX_IVAR'] + pmraivar = objects['PMRA_IVAR'] + parallaxivar = objects['PARALLAX_IVAR'] # ADM derive the parallax/parallax_error, but set to 0 where the error is bad parallaxovererror = np.where(parallaxivar > 0., parallax*np.sqrt(parallaxivar), 0.) @@ -1722,34 +1715,27 @@ def _prepare_gaia(objects, colnames=None, gaiaonly=False): if np.sum(notzero) > 0: parallaxerr[notzero] = 1 / np.sqrt(parallaxivar[notzero]) - prefix = "GAIA_" - if gaiaonly: - prefix = "" - gaiagmag = objects[prefix + "PHOT_G_MEAN_MAG"] - gaiabmag = objects[prefix + "PHOT_BP_MEAN_MAG"] - gaiarmag = objects[prefix + "PHOT_RP_MEAN_MAG"] - gaiaaen = objects[prefix + "ASTROMETRIC_EXCESS_NOISE"] + gaiagmag = objects["GAIA_PHOT_G_MEAN_MAG"] + gaiabmag = objects["GAIA_PHOT_BP_MEAN_MAG"] + gaiarmag = objects["GAIA_PHOT_RP_MEAN_MAG"] + gaiaaen = objects["GAIA_ASTROMETRIC_EXCESS_NOISE"] # ADM a mild hack, as GAIA_DUPLICATED_SOURCE was a 0/1 integer at some point. - gaiadupsource = objects[prefix + "DUPLICATED_SOURCE"] + gaiadupsource = objects["GAIA_DUPLICATED_SOURCE"] if issubclass(gaiadupsource.dtype.type, np.integer): if len(set(np.atleast_1d(gaiadupsource)) - set([0, 1])) == 0: - gaiadupsource = objects[prefix + "DUPLICATED_SOURCE"].astype(bool) + gaiadupsource = objects["GAIA_DUPLICATED_SOURCE"].astype(bool) # For BGS target selection. # ADM first guard against FLUX_R < 0 (I've checked this generates # ADM the same set of targets as Grr = NaN). - if gaiaonly: - Grr = np.zeros_like(objects[prefix + "PHOT_G_MEAN_MAG"]) - Grr = np.nan + Grr = gaiagmag - 22.5 + 2.5*np.log10(1e-16) + ii = objects['FLUX_R'] > 0 + # ADM catch the case where Grr is a scalar. + if isinstance(Grr, np.float): + if ii: + Grr = gaiagmag - 22.5 + 2.5*np.log10(objects['FLUX_R']) else: - Grr = gaiagmag - 22.5 + 2.5*np.log10(1e-16) - ii = objects['FLUX_R'] > 0 - # ADM catch the case where Grr is a scalar. - if isinstance(Grr, np.float): - if ii: - Grr = gaiagmag - 22.5 + 2.5*np.log10(objects['FLUX_R']) - else: - Grr[ii] = gaiagmag[ii] - 22.5 + 2.5*np.log10(objects['FLUX_R'][ii]) + Grr[ii] = gaiagmag[ii] - 22.5 + 2.5*np.log10(objects['FLUX_R'][ii]) # ADM If proper motion is not NaN, 31 parameters were solved for # ADM in Gaia astrometry. Or, gaiaparamssolved should be 3 for NaNs). @@ -2258,7 +2244,7 @@ def apply_cuts_gaia(numproc=4, survey='main', nside=None, pixlist=None): dec = gaiaobjs["DEC"] gaia, pmra, pmdec, parallax, parallaxovererror, parallaxerr, gaiagmag, gaiabmag, \ gaiarmag, gaiaaen, gaiadupsource, Grr, gaiaparamssolved, gaiabprpfactor, \ - gaiasigma5dmax, galb = _prepare_gaia(objects, gaiaonly=True) + gaiasigma5dmax, galb = _prepare_gaia(gaiaobjs) # ADM determine if an object is a BACKUP target. primary = np.ones_like(gaiaobjs, dtype=bool) From 823412566d0e9e74e18f7176ad325ba6487f3cff Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 8 Jan 2021 16:51:22 -0800 Subject: [PATCH 24/30] begin debugging: Need to return the full suite of Gaia parameters --- py/desitarget/cuts.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/py/desitarget/cuts.py b/py/desitarget/cuts.py index 028294687..affee2144 100644 --- a/py/desitarget/cuts.py +++ b/py/desitarget/cuts.py @@ -1704,8 +1704,8 @@ def _prepare_gaia(objects, colnames=None): pmra = objects['PMRA'] pmdec = objects['PMDEC'] parallax = objects['PARALLAX'] - pmraivar = objects['PMRA_IVAR'] parallaxivar = objects['PARALLAX_IVAR'] + pmraivar = objects['PMRA_IVAR'] # ADM derive the parallax/parallax_error, but set to 0 where the error is bad parallaxovererror = np.where(parallaxivar > 0., parallax*np.sqrt(parallaxivar), 0.) @@ -1715,15 +1715,15 @@ def _prepare_gaia(objects, colnames=None): if np.sum(notzero) > 0: parallaxerr[notzero] = 1 / np.sqrt(parallaxivar[notzero]) - gaiagmag = objects["GAIA_PHOT_G_MEAN_MAG"] - gaiabmag = objects["GAIA_PHOT_BP_MEAN_MAG"] - gaiarmag = objects["GAIA_PHOT_RP_MEAN_MAG"] - gaiaaen = objects["GAIA_ASTROMETRIC_EXCESS_NOISE"] + gaiagmag = objects['GAIA_PHOT_G_MEAN_MAG'] + gaiabmag = objects['GAIA_PHOT_BP_MEAN_MAG'] + gaiarmag = objects['GAIA_PHOT_RP_MEAN_MAG'] + gaiaaen = objects['GAIA_ASTROMETRIC_EXCESS_NOISE'] # ADM a mild hack, as GAIA_DUPLICATED_SOURCE was a 0/1 integer at some point. - gaiadupsource = objects["GAIA_DUPLICATED_SOURCE"] + gaiadupsource = objects['GAIA_DUPLICATED_SOURCE'] if issubclass(gaiadupsource.dtype.type, np.integer): if len(set(np.atleast_1d(gaiadupsource)) - set([0, 1])) == 0: - gaiadupsource = objects["GAIA_DUPLICATED_SOURCE"].astype(bool) + gaiadupsource = objects['GAIA_DUPLICATED_SOURCE'].astype(bool) # For BGS target selection. # ADM first guard against FLUX_R < 0 (I've checked this generates @@ -2234,7 +2234,7 @@ def apply_cuts_gaia(numproc=4, survey='main', nside=None, pixlist=None): # ADM or are north of dec=-30. gaiaobjs = all_gaia_in_tiles(maglim=19, numproc=numproc, allsky=True, mindec=-30, mingalb=0, addobjid=True, - nside=nside, pixlist=pixlist) + nside=nside, pixlist=pixlist, addparams=True) # ADM the convenience function we use adds an empty TARGETID # ADM field which we need to remove before finalizing. gaiaobjs = rfn.drop_fields(gaiaobjs, "TARGETID") From 0fbaae23347b5506ad9edd16888f6fe6df8abf88 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 8 Jan 2021 16:52:30 -0800 Subject: [PATCH 25/30] minor textual changes to check against master --- py/desitarget/cuts.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/py/desitarget/cuts.py b/py/desitarget/cuts.py index affee2144..5d9d070bb 100644 --- a/py/desitarget/cuts.py +++ b/py/desitarget/cuts.py @@ -1703,9 +1703,9 @@ def _prepare_gaia(objects, colnames=None): gaia = (objects['REF_CAT'] == b'G2') | (objects['REF_CAT'] == 'G2') pmra = objects['PMRA'] pmdec = objects['PMDEC'] + pmraivar = objects['PMRA_IVAR'] parallax = objects['PARALLAX'] parallaxivar = objects['PARALLAX_IVAR'] - pmraivar = objects['PMRA_IVAR'] # ADM derive the parallax/parallax_error, but set to 0 where the error is bad parallaxovererror = np.where(parallaxivar > 0., parallax*np.sqrt(parallaxivar), 0.) @@ -1714,7 +1714,6 @@ def _prepare_gaia(objects, colnames=None): notzero = parallaxivar > 0 if np.sum(notzero) > 0: parallaxerr[notzero] = 1 / np.sqrt(parallaxivar[notzero]) - gaiagmag = objects['GAIA_PHOT_G_MEAN_MAG'] gaiabmag = objects['GAIA_PHOT_BP_MEAN_MAG'] gaiarmag = objects['GAIA_PHOT_RP_MEAN_MAG'] From e0643d84a31ca3825be3b99281869eeb4105c7a2 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 8 Jan 2021 17:35:00 -0800 Subject: [PATCH 26/30] continue debugging; there are now standards in the MWS mask that have priorities of -1 (as calib targets); import needed modules --- py/desitarget/cuts.py | 2 +- py/desitarget/sv1/sv1_cuts.py | 9 ++++++-- py/desitarget/targets.py | 30 ++++++++++++++++----------- py/desitarget/test/test_priorities.py | 7 ++++--- 4 files changed, 30 insertions(+), 18 deletions(-) diff --git a/py/desitarget/cuts.py b/py/desitarget/cuts.py index 5d9d070bb..a058a7711 100644 --- a/py/desitarget/cuts.py +++ b/py/desitarget/cuts.py @@ -2259,7 +2259,7 @@ def apply_cuts_gaia(numproc=4, survey='main', nside=None, pixlist=None): gaiabprpfactor=gaiabprpfactor, gaiasigma5dmax=gaiasigma5dmax, gaiagmag=gaiagmag, gaiabmag=gaiabmag, gaiarmag=gaiarmag, gaiadupsource=gaiadupsource, gaiaparamssolved=gaiaparamssolved, - primary=primary) + primary=primary, nside=nside) # ADM Construct the target flag bits. mws_target = backup_bright * mws_mask.BACKUP_BRIGHT diff --git a/py/desitarget/sv1/sv1_cuts.py b/py/desitarget/sv1/sv1_cuts.py index 39523bb00..1312d06d5 100644 --- a/py/desitarget/sv1/sv1_cuts.py +++ b/py/desitarget/sv1/sv1_cuts.py @@ -17,10 +17,12 @@ from time import time from pkg_resources import resource_filename +import healpy as hp +import fitsio from desitarget.cuts import _getColors, _psflike, _check_BGS_targtype_sv from desitarget.cuts import shift_photo_north -from desitarget.gaiamatch import is_in_Galaxy +from desitarget.gaiamatch import is_in_Galaxy, find_gaia_files_hp from desitarget.geomask import imaging_mask # ADM set up the DESI default logger @@ -35,7 +37,7 @@ def isGAIA_STD(ra=None, dec=None, galb=None, gaiaaen=None, pmra=None, pmdec=None parallax=None, parallaxovererror=None, gaiabprpfactor=None, gaiasigma5dmax=None, gaiagmag=None, gaiabmag=None, gaiarmag=None, gaiadupsource=None, gaiaparamssolved=None, - primary=None, test=False): + primary=None, test=False, nside=2): """Standards based solely on Gaia data. Parameters @@ -43,6 +45,9 @@ def isGAIA_STD(ra=None, dec=None, galb=None, gaiaaen=None, pmra=None, pmdec=None test : :class:`bool`, optional, defaults to ``False`` If ``True``, then we're running unit tests and don't have to find and read every possible Gaia file. + nside : :class:`int`, optional, defaults to 2 + (NESTED) HEALPix nside, if targets are being parallelized. + The default of 2 should be benign for serial processing. see :func:`~desitarget.sv1.sv1_cuts.set_target_bits` for parameters. diff --git a/py/desitarget/targets.py b/py/desitarget/targets.py index 4c31b1871..712c48029 100644 --- a/py/desitarget/targets.py +++ b/py/desitarget/targets.py @@ -686,23 +686,29 @@ def calc_priority(targets, zcat, obscon, state=False): # MWS targets. if mws_target in targets.dtype.names: + # ADM set initialstate of CALIB for potential calibration targets. + stdnames = ('GAIA_STD_FAINT', 'GAIA_STD_WD', 'GAIA_STD_BRIGHT') for name in mws_mask.names(): # ADM only update priorities for passed observing conditions. pricon = obsconditions.mask(mws_mask[name].obsconditions) if (obsconditions.mask(obscon) & pricon) != 0: ii = (targets[mws_target] & mws_mask[name]) != 0 - for sbool, sname in zip( - [unobs, done, zgood, zwarn], - ["UNOBS", "DONE", "MORE_ZGOOD", "MORE_ZWARN"] - ): - # ADM update priorities and target states. - Mxp = mws_mask[name].priorities[sname] - # ADM update states BEFORE changing priorities. - ts = "{}|{}".format("MWS", sname) - target_state[ii & sbool] = np.where( - priority[ii & sbool] < Mxp, ts, target_state[ii & sbool]) - priority[ii & sbool] = np.where( - priority[ii & sbool] < Mxp, Mxp, priority[ii & sbool]) + # ADM standards have no priority. + if name in stdnames: + target_state[ii] = "CALIB" + else: + for sbool, sname in zip( + [unobs, done, zgood, zwarn], + ["UNOBS", "DONE", "MORE_ZGOOD", "MORE_ZWARN"] + ): + # ADM update priorities and target states. + Mxp = mws_mask[name].priorities[sname] + # ADM update states BEFORE changing priorities. + ts = "{}|{}".format("MWS", sname) + target_state[ii & sbool] = np.where( + priority[ii & sbool] < Mxp, ts, target_state[ii & sbool]) + priority[ii & sbool] = np.where( + priority[ii & sbool] < Mxp, Mxp, priority[ii & sbool]) # ADM Secondary targets. if scnd_target in targets.dtype.names: diff --git a/py/desitarget/test/test_priorities.py b/py/desitarget/test/test_priorities.py index 5c84424fc..d79d6496e 100644 --- a/py/desitarget/test/test_priorities.py +++ b/py/desitarget/test/test_priorities.py @@ -132,8 +132,9 @@ def test_priorities(self): # ADM should never trump the other bits. bgs_names = [name for name in bgs_mask.names() if 'NORTH' not in name and 'SOUTH' not in name] - mws_names = [name for name in mws_mask.names() if 'NORTH' not in name - and 'SOUTH' not in name and 'BACKUP' not in name] + mws_names = [name for name in mws_mask.names() if + 'NORTH' not in name and 'SOUTH' not in name and + 'BACKUP' not in name and 'STD' not in name] lowest_mws_priority_unobs = [mws_mask[n].priorities['UNOBS'] for n in mws_names] @@ -172,7 +173,7 @@ def test_mask_priorities(self): for mask in [desi_mask, bgs_mask, mws_mask]: for name in mask.names(): if ( - name.startswith('STD') or name.endswith('BRIGHT_OBJECT') or name in + 'STD' in name or name.endswith('BRIGHT_OBJECT') or name in ['BGS_ANY', 'MWS_ANY', 'SCND_ANY', 'SKY', 'SUPP_SKY', 'NO_TARGET'] ): self.assertEqual(mask[name].priorities, {}, 'mask.{} has priorities?'.format(name)) From 911a7b3679c20885e1251544803610f7e784662a Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 8 Jan 2021 18:57:02 -0800 Subject: [PATCH 27/30] continue debugging; correct parameter names, import more needed modules --- py/desitarget/cuts.py | 6 +++--- py/desitarget/sv1/sv1_cuts.py | 11 +++++++---- py/desitarget/targets.py | 4 ++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/py/desitarget/cuts.py b/py/desitarget/cuts.py index a058a7711..f1c3e79b3 100644 --- a/py/desitarget/cuts.py +++ b/py/desitarget/cuts.py @@ -2265,9 +2265,9 @@ def apply_cuts_gaia(numproc=4, survey='main', nside=None, pixlist=None): mws_target = backup_bright * mws_mask.BACKUP_BRIGHT mws_target |= backup_faint * mws_mask.BACKUP_FAINT mws_target |= backup_very_faint * mws_mask.BACKUP_VERY_FAINT - mws_target |= std_faint * mws_mask.STD_FAINT - mws_target |= std_bright * mws_mask.STD_BRIGHT - mws_target |= std_wd * mws_mask.STD_WD + mws_target |= std_faint * mws_mask.GAIA_STD_FAINT + mws_target |= std_bright * mws_mask.GAIA_STD_BRIGHT + mws_target |= std_wd * mws_mask.GAIA_STD_WD bgs_target = np.zeros_like(mws_target) diff --git a/py/desitarget/sv1/sv1_cuts.py b/py/desitarget/sv1/sv1_cuts.py index 1312d06d5..ee92c70f0 100644 --- a/py/desitarget/sv1/sv1_cuts.py +++ b/py/desitarget/sv1/sv1_cuts.py @@ -20,6 +20,9 @@ import healpy as hp import fitsio +import astropy.units as u +from astropy.coordinates import SkyCoord + from desitarget.cuts import _getColors, _psflike, _check_BGS_targtype_sv from desitarget.cuts import shift_photo_north from desitarget.gaiamatch import is_in_Galaxy, find_gaia_files_hp @@ -102,7 +105,7 @@ def isGAIA_STD(ra=None, dec=None, galb=None, gaiaaen=None, pmra=None, pmdec=None gbp = gaiagmag - gaiabmag std &= bprp > 0.2 std &= bprp < 0.9 - std &= gbp > -1*bprp/2.0 + std &= gbp > -1.*bprp/2.0 std &= gbp < 0.3-bprp/2.0 # ADM remove any sources that have neighbors in Gaia within 3.5"... @@ -119,7 +122,7 @@ def isGAIA_STD(ra=None, dec=None, galb=None, gaiaaen=None, pmra=None, pmdec=None gaiacols = ["RA", "DEC", "PHOT_G_MEAN_MAG", "PHOT_RP_MEAN_MAG"] for i, fn in enumerate(fns): if i % 25 == 0: - log.info("Read {}/{} files for STD_DITHER_GAIA...t={:.1f}s" + log.info("Read {}/{} files for Gaia-only standards...t={:.1f}s" .format(i, len(fns), time()-start)) try: gaiaobjs.append(fitsio.read(fn, columns=gaiacols)) @@ -141,8 +144,8 @@ def isGAIA_STD(ra=None, dec=None, galb=None, gaiaaen=None, pmra=None, pmdec=None idgaia, idstd = idgaia[d2d > 0], idstd[d2d > 0] # ADM remove matches within 5 mags of a Gaia source. badmag = ( - (gmag[ii_true][idstd] + 5 > gaiaobjs["PHOT_G_MEAN_MAG"][idgaia]) | - (rmag[ii_true][idstd] + 5 > gaiaobjs["PHOT_RP_MEAN_MAG"][idgaia])) + (gaiagmag[ii_true][idstd] + 5 > gaiaobjs["PHOT_G_MEAN_MAG"][idgaia]) | + (gaiarmag[ii_true][idstd] + 5 > gaiaobjs["PHOT_RP_MEAN_MAG"][idgaia])) std[ii_true[idstd][badmag]] = False # ADM add the brightness cuts in Gaia G-band. diff --git a/py/desitarget/targets.py b/py/desitarget/targets.py index 712c48029..2adf59310 100644 --- a/py/desitarget/targets.py +++ b/py/desitarget/targets.py @@ -607,7 +607,7 @@ def calc_priority(targets, zcat, obscon, state=False): # DESI dark time targets. if survey != 'cmx': if desi_target in targets.dtype.names: - # ADM set initialstate of CALIB for potential calibration targets. + # ADM set initial state of CALIB for potential calibration targets. names = ('SKY', 'BAD_SKY', 'SUPP_SKY', 'STD_FAINT', 'STD_WD', 'STD_BRIGHT') for name in names: @@ -686,7 +686,7 @@ def calc_priority(targets, zcat, obscon, state=False): # MWS targets. if mws_target in targets.dtype.names: - # ADM set initialstate of CALIB for potential calibration targets. + # ADM set initial state of CALIB for potential calibration targets. stdnames = ('GAIA_STD_FAINT', 'GAIA_STD_WD', 'GAIA_STD_BRIGHT') for name in mws_mask.names(): # ADM only update priorities for passed observing conditions. From 424430255a6f302cfa20df0296821f19090347c6 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 8 Jan 2021 19:21:44 -0800 Subject: [PATCH 28/30] Duplicate Gaia standard stars code for the Main Survey; update changes docs --- doc/changes.rst | 2 + py/desitarget/cuts.py | 124 ++++++++++++++++++++++++++++++++++ py/desitarget/sv1/sv1_cuts.py | 2 +- 3 files changed, 127 insertions(+), 1 deletion(-) diff --git a/doc/changes.rst b/doc/changes.rst index 71549e6a3..e672d991d 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -6,6 +6,8 @@ desitarget Change Log ------------------- * First run of secondaries with real target files [`PR #669`_]. Includes: + * Add Gaia-only standard stars to the MWS masks for SV, Main Survey: + * `GAIA_STD_FAINT`, `GAIA_STD_BRIGHT`, `GAIA_STD_WD`. * General optimization, updating and debugging of the secondary code. * Get `TARGETIDs` from the input sweeps, not just the actual targets. * Add the first full bitmask for the SV1 secondary target files. diff --git a/py/desitarget/cuts.py b/py/desitarget/cuts.py index f1c3e79b3..f9a4391bc 100644 --- a/py/desitarget/cuts.py +++ b/py/desitarget/cuts.py @@ -138,6 +138,130 @@ def shift_photo_north(gflux=None, rflux=None, zflux=None): return gshift, rshift, zshift +def isGAIA_STD(ra=None, dec=None, galb=None, gaiaaen=None, pmra=None, pmdec=None, + parallax=None, parallaxovererror=None, gaiabprpfactor=None, + gaiasigma5dmax=None, gaiagmag=None, gaiabmag=None, gaiarmag=None, + gaiadupsource=None, gaiaparamssolved=None, + primary=None, test=False, nside=2): + """Standards based solely on Gaia data. + + Parameters + ---------- + test : :class:`bool`, optional, defaults to ``False`` + If ``True``, then we're running unit tests and don't have to + find and read every possible Gaia file. + nside : :class:`int`, optional, defaults to 2 + (NESTED) HEALPix nside, if targets are being parallelized. + The default of 2 should be benign for serial processing. + + see :func:`~desitarget.sv1.sv1_cuts.set_target_bits` for parameters. + + Returns + ------- + :class:`array_like` + ``True`` if the object is a bright "GAIA_STD_FAINT" target. + :class:`array_like` + ``True`` if the object is a faint "GAIA_STD_BRIGHT" target. + :class:`array_like` + ``True`` if the object is a white dwarf "GAIA_STD_WD" target. + + Notes + ----- + - Current version (01/08/21) is version XXX on `the wiki`_. + """ + if primary is None: + primary = np.ones_like(gaiagmag, dtype='?') + + # ADM restrict all classes to dec >= -30. + primary &= dec >= -30. + std = primary.copy() + + # ADM the regular "standards" codes need to know whether something has + # ADM a Gaia match. Here, everything is a Gaia match. + gaia = np.ones_like(gaiagmag, dtype='?') + + # ADM determine the Gaia-based white dwarf standards. + std_wd = isMWS_WD( + primary=primary, gaia=gaia, galb=galb, astrometricexcessnoise=gaiaaen, + pmra=pmra, pmdec=pmdec, parallax=parallax, + parallaxovererror=parallaxovererror, photbprpexcessfactor=gaiabprpfactor, + astrometricsigma5dmax=gaiasigma5dmax, gaiagmag=gaiagmag, + gaiabmag=gaiabmag, gaiarmag=gaiarmag + ) + + # ADM apply the Gaia quality cuts for standards. + std &= isSTD_gaia(primary=primary, gaia=gaia, astrometricexcessnoise=gaiaaen, + pmra=pmra, pmdec=pmdec, parallax=parallax, + dupsource=gaiadupsource, paramssolved=gaiaparamssolved, + gaiagmag=gaiagmag, gaiabmag=gaiabmag, gaiarmag=gaiarmag) + + # ADM restrict to point sources. + ispsf = np.logical_or( + (gaiagmag <= 19.) * (gaiaaen < 10.**0.5), + (gaiagmag >= 19.) * (gaiaaen < 10.**(0.5 + 0.2*(gaiagmag - 19.))) + ) + std &= ispsf + + # ADM apply the Gaia color cuts for standards. + bprp = gaiabmag - gaiarmag + gbp = gaiagmag - gaiabmag + std &= bprp > 0.2 + std &= bprp < 0.9 + std &= gbp > -1.*bprp/2.0 + std &= gbp < 0.3-bprp/2.0 + + # ADM remove any sources that have neighbors in Gaia within 3.5"... + # ADM for speed, run only sources for which std is still True. + log.info("Isolating Gaia-only standards...t={:.1f}s".format(time()-start)) + ii_true = np.where(std)[0] + if len(ii_true) > 0: + # ADM determine the pixels of interest. + theta, phi = np.radians(90-dec), np.radians(ra) + pixlist = list(set(hp.ang2pix(nside, theta, phi, nest=True))) + # ADM read in the necessary Gaia files. + fns = find_gaia_files_hp(nside, pixlist, neighbors=True) + gaiaobjs = [] + gaiacols = ["RA", "DEC", "PHOT_G_MEAN_MAG", "PHOT_RP_MEAN_MAG"] + for i, fn in enumerate(fns): + if i % 25 == 0: + log.info("Read {}/{} files for Gaia-only standards...t={:.1f}s" + .format(i, len(fns), time()-start)) + try: + gaiaobjs.append(fitsio.read(fn, columns=gaiacols)) + except OSError: + if test: + pass + else: + msg = "failed to find or open the following file: (ffopen) " + msg += fn + log.critical(msg) + raise OSError + gaiaobjs = np.concatenate(gaiaobjs) + # ADM match the standards to the broader Gaia sources at 3.5". + matchrad = 3.5*u.arcsec + cstd = SkyCoord(ra[ii_true]*u.degree, dec[ii_true]*u.degree) + cgaia = SkyCoord(gaiaobjs["RA"]*u.degree, gaiaobjs["DEC"]*u.degree) + idstd, idgaia, d2d, _ = cgaia.search_around_sky(cstd, matchrad) + # ADM remove source matches with d2d=0 (i.e. the source itself!). + idgaia, idstd = idgaia[d2d > 0], idstd[d2d > 0] + # ADM remove matches within 5 mags of a Gaia source. + badmag = ( + (gaiagmag[ii_true][idstd] + 5 > gaiaobjs["PHOT_G_MEAN_MAG"][idgaia]) | + (gaiarmag[ii_true][idstd] + 5 > gaiaobjs["PHOT_RP_MEAN_MAG"][idgaia])) + std[ii_true[idstd][badmag]] = False + + # ADM add the brightness cuts in Gaia G-band. + std_bright = std.copy() + std_bright &= gaiagmag >= 15 + std_bright &= gaiagmag < 18 + + std_faint = std.copy() + std_faint &= gaiagmag >= 16 + std_faint &= gaiagmag < 19 + + return std_faint, std_bright, std_wd + + def isBACKUP(ra=None, dec=None, gaiagmag=None, primary=None): """BACKUP targets based on Gaia magnitudes. diff --git a/py/desitarget/sv1/sv1_cuts.py b/py/desitarget/sv1/sv1_cuts.py index ee92c70f0..a193e0595 100644 --- a/py/desitarget/sv1/sv1_cuts.py +++ b/py/desitarget/sv1/sv1_cuts.py @@ -93,7 +93,7 @@ def isGAIA_STD(ra=None, dec=None, galb=None, gaiaaen=None, pmra=None, pmdec=None dupsource=gaiadupsource, paramssolved=gaiaparamssolved, gaiagmag=gaiagmag, gaiabmag=gaiabmag, gaiarmag=gaiarmag) - # ADM restrict to point sources + # ADM restrict to point sources. ispsf = np.logical_or( (gaiagmag <= 19.) * (gaiaaen < 10.**0.5), (gaiagmag >= 19.) * (gaiaaen < 10.**(0.5 + 0.2*(gaiagmag - 19.))) From b109cdb62051c542bd7612f8f804c9daa6316943 Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 8 Jan 2021 19:49:56 -0800 Subject: [PATCH 29/30] update unit tests and fix sphinx warnings --- py/desitarget/cuts.py | 27 +++++++++++++++++++-------- py/desitarget/sv1/sv1_cuts.py | 3 +-- py/desitarget/test/test_sv.py | 4 ++-- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/py/desitarget/cuts.py b/py/desitarget/cuts.py index f9a4391bc..750d2655c 100644 --- a/py/desitarget/cuts.py +++ b/py/desitarget/cuts.py @@ -19,17 +19,20 @@ import numbers import sys +import fitsio import numpy as np import healpy as hp from pkg_resources import resource_filename import numpy.lib.recfunctions as rfn from importlib import import_module +import astropy.units as u +from astropy.coordinates import SkyCoord from astropy.table import Table, Row from desitarget import io from desitarget.internal import sharedmem -from desitarget.gaiamatch import match_gaia_to_primary +from desitarget.gaiamatch import match_gaia_to_primary, find_gaia_files_hp from desitarget.gaiamatch import pop_gaia_coords, pop_gaia_columns from desitarget.gaiamatch import gaia_dr_from_ref_cat, is_in_Galaxy from desitarget.targets import finalize, resolve @@ -154,8 +157,6 @@ def isGAIA_STD(ra=None, dec=None, galb=None, gaiaaen=None, pmra=None, pmdec=None (NESTED) HEALPix nside, if targets are being parallelized. The default of 2 should be benign for serial processing. - see :func:`~desitarget.sv1.sv1_cuts.set_target_bits` for parameters. - Returns ------- :class:`array_like` @@ -168,6 +169,7 @@ def isGAIA_STD(ra=None, dec=None, galb=None, gaiaaen=None, pmra=None, pmdec=None Notes ----- - Current version (01/08/21) is version XXX on `the wiki`_. + - See :func:`~desitarget.cuts.set_target_bits` for other parameters. """ if primary is None: primary = np.ones_like(gaiagmag, dtype='?') @@ -491,7 +493,7 @@ def notinELG_mask(maskbits=None, gsnr=None, rsnr=None, zsnr=None, def isELG_colors(gflux=None, rflux=None, zflux=None, w1flux=None, w2flux=None, south=True, primary=None): """Color cuts for ELG target selection classes - (see, e.g., :func:`desitarget.cuts.set_target_bits` for parameters). + (see, e.g., :func:`~desitarget.cuts.set_target_bits` for parameters). """ if primary is None: primary = np.ones_like(rflux, dtype='?') @@ -2292,7 +2294,8 @@ def set_target_bits(photsys_north, photsys_south, obs_rflux, return desi_target, bgs_target, mws_target -def apply_cuts_gaia(numproc=4, survey='main', nside=None, pixlist=None): +def apply_cuts_gaia(numproc=4, survey='main', nside=None, pixlist=None, + test=False): """Gaia-only-based target selection, return target mask arrays. Parameters @@ -2309,6 +2312,10 @@ def apply_cuts_gaia(numproc=4, survey='main', nside=None, pixlist=None): Only return targets in a set of (NESTED) HEALpixels at `nside`. Useful for parallelizing, as input files will only be processed if they touch a pixel in the passed list. + test : :class:`bool`, optional, defaults to ``False`` + If ``True``, then we're running unit tests and don't have to find + and read every possible Gaia file when calling + :func:`~desitarget.cuts.apply_cuts_gaia`. Returns ------- @@ -2383,7 +2390,7 @@ def apply_cuts_gaia(numproc=4, survey='main', nside=None, pixlist=None): gaiabprpfactor=gaiabprpfactor, gaiasigma5dmax=gaiasigma5dmax, gaiagmag=gaiagmag, gaiabmag=gaiabmag, gaiarmag=gaiarmag, gaiadupsource=gaiadupsource, gaiaparamssolved=gaiaparamssolved, - primary=primary, nside=nside) + primary=primary, nside=nside, test=test) # ADM Construct the target flag bits. mws_target = backup_bright * mws_mask.BACKUP_BRIGHT @@ -2546,7 +2553,7 @@ def select_targets(infiles, numproc=4, qso_selection='randomforest', extra=None, radecbox=None, radecrad=None, mask=True, tcnames=["ELG", "QSO", "LRG", "MWS", "BGS", "STD"], survey='main', resolvetargs=True, backup=True, - return_infiles=False): + return_infiles=False, test=False): """Process input files in parallel to select targets. Parameters @@ -2602,6 +2609,10 @@ def select_targets(infiles, numproc=4, qso_selection='randomforest', If ``True``, also return the actual files from `infile` processed. Useful when running with `pixlist`, `radecbox` or `radecrad` to see which files were actually required. + test : :class:`bool`, optional, defaults to ``False`` + If ``True``, then we're running unit tests and don't have to find + and read every possible Gaia file when calling + :func:`~desitarget.cuts.apply_cuts_gaia`. Returns ------- @@ -2770,7 +2781,7 @@ def _update_status(result): # ADM set the target bits that are based only on Gaia. gaia_desi_target, gaia_bgs_target, gaia_mws_target, gaiaobjs = \ apply_cuts_gaia(numproc=numproc4, survey=survey, nside=nside, - pixlist=pixlist) + pixlist=pixlist, test=test) # ADM it's possible that somebody could pass HEALPixels that # ADM contain no additional targets. diff --git a/py/desitarget/sv1/sv1_cuts.py b/py/desitarget/sv1/sv1_cuts.py index a193e0595..fd99feb3f 100644 --- a/py/desitarget/sv1/sv1_cuts.py +++ b/py/desitarget/sv1/sv1_cuts.py @@ -52,8 +52,6 @@ def isGAIA_STD(ra=None, dec=None, galb=None, gaiaaen=None, pmra=None, pmdec=None (NESTED) HEALPix nside, if targets are being parallelized. The default of 2 should be benign for serial processing. - see :func:`~desitarget.sv1.sv1_cuts.set_target_bits` for parameters. - Returns ------- :class:`array_like` @@ -65,6 +63,7 @@ def isGAIA_STD(ra=None, dec=None, galb=None, gaiaaen=None, pmra=None, pmdec=None Notes ----- + - See :func:`~desitarget.cuts.set_target_bits` for other parameters. - Current version (01/08/21) is version XXX on `the SV wiki`_. """ if primary is None: diff --git a/py/desitarget/test/test_sv.py b/py/desitarget/test/test_sv.py index ebd5d2211..70980c986 100644 --- a/py/desitarget/test/test_sv.py +++ b/py/desitarget/test/test_sv.py @@ -56,7 +56,7 @@ def test_sv_cuts(self): # ADM set backup to False as the Gaia unit test # ADM files only cover a limited pixel range. targets = cuts.select_targets(filelist, survey=survey, - backup=False) + backup=False, test=True) for col in desicol, bgscol, mwscol: self.assertTrue(col in targets.dtype.names) self.assertEqual(len(targets), np.count_nonzero(targets[desicol])) @@ -71,7 +71,7 @@ def test_sv_cuts(self): for filelist in self.sweepfiles: # ADM also test the backup targets in the pixels covered # ADM by the Gaia unit test files. - targets = cuts.select_targets(filelist, survey=survey, + targets = cuts.select_targets(filelist, survey=survey, test=True, nside=self.nside, pixlist=self.pix) for col in desicol, bgscol, mwscol: self.assertTrue(col in targets.dtype.names) From fae721c783749fc4af4414487fd88e502159f66d Mon Sep 17 00:00:00 2001 From: Adam Myers Date: Fri, 8 Jan 2021 20:21:12 -0800 Subject: [PATCH 30/30] update the Main Survey unit tests for the new GAIA_STD classes, too --- py/desitarget/test/test_cuts.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/py/desitarget/test/test_cuts.py b/py/desitarget/test/test_cuts.py index 6581c9a98..93d7eba2b 100644 --- a/py/desitarget/test/test_cuts.py +++ b/py/desitarget/test/test_cuts.py @@ -362,8 +362,9 @@ def test_backup(self): # ADM BACKUP targets can only run on the sweep files. for filelist in self.sweepfiles: # ADM limit to pixels covered in the Gaia unit test files. - targets = cuts.select_targets(filelist, numproc=1, tcnames=tc, - nside=self.nside, pixlist=self.pix) + targets = cuts.select_targets( + filelist, numproc=1, tcnames=tc, test=True, nside=self.nside, + pixlist=self.pix) self.assertTrue('DESI_TARGET' in targets.dtype.names) self.assertTrue('BGS_TARGET' in targets.dtype.names) self.assertTrue('MWS_TARGET' in targets.dtype.names)