Merge 3a914d1 into 700c90a

bin/select_secondary

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+
+import sys
+import fitsio
+import numpy as np
+import argparse
+from desitarget.secondary import select_secondary, _get_scxdir
+from desitarget.targets import main_cmx_or_sv
+from desitarget import io
+import os
+from glob import glob
+from time import time
+time0 = time()
+
+from desiutil.log import get_logger
+log = get_logger()
+
+import multiprocessing
+nproc = multiprocessing.cpu_count() // 2
+
+from argparse import ArgumentParser
+ap = ArgumentParser(description='Generate file of secondary-only targets from $SECONDARY_DIR, write matches to primary targets back to $SECONDARY_DIR')
+ap.add_argument("surveydir",
+                help="Base directory of primary target files (e.g. '/project/projectdirs/desi/target/catalogs/dr7.1/0.22.0/' at NERSC). "+ 
+                "All files that contain the string 'target' or 'targets' in their names will be considered a target file." +
+                "Whether the file is an SV file is derived from the DESI_TARGET column in the first file detected in this directory.")
+ap.add_argument("dest",
+                help="Output secondary-only targets file (e.g. '/project/projectdirs/desi/target/catalogs/secondary-dr6-0.20.0.fits' at NERSC)")
+ap.add_argument('-n', "--numproc", type=int,
+                help='number of concurrent processes to use (defaults to [{}])'.format(nproc),
+                default=nproc)
+ap.add_argument("-s", "--separation", type=float, default=1.,
+                help='Angular separation at which to match primary and secondary targets. Defaults to [1.]')
+ap.add_argument("--scnddir",
+                help="Base directory of secondary target files (e.g. '/project/projectdirs/desi/target/secondary' at NERSC). "+
+                "Defaults to SECONDARY_DIR environment variable.")
+
+ns = ap.parse_args()
+
+infiles = io.list_targetfiles(ns.surveydir)
+if len(infiles) == 0:
+    log.critical('no target files found')
+    sys.exit(1)
+
+log.info('running on {} processors...t={:.1f}mins'.format(ns.numproc, (time()-time0)/60.))
+
+# ADM check if this is an SV or main survey file.
+chk = fitsio.read(infiles[0],rows=0)
+cols, mx, surv = main_cmx_or_sv(chk, scnd=True)
+log.info('running on the {} survey...'.format(surv))
+
+# ADM retrieve the SECONDARY_DIR, in case it wasn't passed and it's stored as an environment variable.
+scxdir = _get_scxdir(ns.scnddir)
+# ADM and augment the scxdir if this is an SV set of primary files.
+if surv != 'main':
+    scxdir = os.path.join(scxdir, surv)
+
+scx = select_secondary(infiles, numproc=ns.numproc, sep=ns.separation, scxdir=scxdir, scnd_mask=mx[3])
+
+# ADM start the header with that of the first primary file, the primary directory and the separation.
+hdr = fitsio.read_header(infiles[0], extension='TARGETS')
+hdr['PRIMDIR'] = ns.surveydir
+hdr['SEP'] = float(ns.separation)
+
+# ADM write out the secondary targets.
+io.write_secondary(ns.dest, scx, primhdr=hdr, scxdir=scxdir)
+
+log.info('{} secondary targets written to {}...t={:.1f}mins'.format(len(scx), ns.dest, (time()-time0)/60.))

doc/changes.rst

@@ -5,7 +5,25 @@ desitarget Change Log
 0.30.2 (unreleased)
 -------------------
 
-* No changes yet.
+* First implementation for secondary targets [`PR #507`_]. Includes:
+    * Framework and design for secondary targeting process.
+    * Works automatically for both Main Survey and SV files.
+    * New bitmasks for secondaries that populate ``SCND_TARGET`` column.
+        * can have any ``PRIORITY_INIT`` and ``NUMOBS_INIT``.
+    * A reserved "veto" bit to categorically reject targets.
+    * Rigorous checking of file formats...
+        * ...and that files correspond to secondary bits.
+    * Example files and file structure (at NERSC) in ``SCND_DIR``.
+        * /project/projectdirs/desi/target/secondary.
+    * Secondary targets are matched to primary targets on RA/Dec.
+        * unless a (per-source) ``OVERRIDE`` column is set to ``True``.
+    * Secondary-primary matches share the primary ``TARGETID``.
+    * Non-matches and overrides have their own ``TARGETID``.
+        * with ``RELEASE == 0``.
+    * Non-override secondary targets are also matched to themselves.
+        * ``TARGETID`` and ``SCND_TARGET`` correspond for matches.
+
+.. _`PR #507`: https://github.com/desihub/desitarget/pull/507
 
 0.30.1 (2019-06-18)
 -------------------
@@ -15,7 +33,7 @@ desitarget Change Log
 * Follow-up PR to `PR #496`_ with two changes and bug fixes [`PR #505`_]:
     * Select QSO targets using random forest by default.
     * Bug fix: Correctly populate ``REF_CAT`` column (needed to correctly set
-      MWS targeting bits).     
+      MWS targeting bits).
 
 .. _`PR #505`: https://github.com/desihub/desitarget/pull/505
 .. _`PR #506`: https://github.com/desihub/desitarget/pull/506
@@ -94,6 +112,7 @@ desitarget Change Log
 
 0.29.0 (2019-03-22)
 -------------------
+
 * Update SV selection for DR8 [`PR #477`_]. Includes:
     * New SV targeting bits for QSOs and LRGs.
     * New SV selection algorithms for QSOs, ELGs and LRGs.
@@ -395,7 +414,6 @@ bit names and selection function names.
 
 .. _`PR #334`: https://github.com/desihub/desitarget/pull/334
 
-
 0.21.0 (2018-07-18)
 -------------------
 
@@ -465,7 +483,6 @@ bit names and selection function names.
 .. _`PR #331`: https://github.com/desihub/desitarget/pull/331
 .. _`PR #332`: https://github.com/desihub/desitarget/pull/332
 
-
 0.20.1 (2018-03-29)
 -------------------
 

py/desitarget/data/targetmask.yaml

@@ -55,7 +55,7 @@ desi_mask:
     #- A bit for another survey is set
     - [BGS_ANY,             60, "Any BGS bit is set", {obsconditions: BRIGHT}]
     - [MWS_ANY,             61, "Any MWS bit is set", {obsconditions: BRIGHT}]
-    - [SECONDARY_ANY,       62, "Any secondary bit is set",
+    - [SCND_ANY     ,       62, "Any secondary bit is set",
         {obsconditions: DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18}]
 
 #- Bright Galaxy Survey
@@ -111,6 +111,18 @@ mws_mask:
     - [MWS_MAIN_RED_NORTH,  12, "MWS magnitude limited red sample tuned for Bok/Mosaic",      {obsconditions: BRIGHT|GRAY|DARK}]
     - [MWS_MAIN_RED_SOUTH,  13, "MWS magnitude limited red sample tuned for DECam",           {obsconditions: BRIGHT|GRAY|DARK}]
 
+#- ADM secondary survey targets bit mask.
+#- ADM note that the 'filenames', here, should all be present in the
+#- ADM directory that corresponds to the $SECONDARY_DIR environment
+#- ADM variable, e.g. $SECONDARY_DIR/backstop.fits for BACKSTOP targets.
+scnd_mask:
+    - [VETO,        0, "Never observe, even if a primary target bit is set",
+       {obsconditions: DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18, filename: 'veto'}]
+    - [BACKSTOP,    1, "Gaia-selected targets of last resort",
+       {obsconditions: DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18, filename: 'backstop'}]
+    - [DR16Q,       2, "Known quasars from the SDSS DR16Q catalog",
+       {obsconditions: DARK|GRAY|BRIGHT|POOR|TWILIGHT12|TWILIGHT18, filename: 'dr16q'}]
+
 #- Observing conditions
 #- These are a bitmask to allow target bits to specify multiple conditions
 #- in which they are allowed to be observed.
@@ -194,7 +206,7 @@ priorities:
         NEAR_BRIGHT_OBJECT: -1
         BGS_ANY: -1
         MWS_ANY: -1
-        SECONDARY_ANY: -1
+        SCND_ANY: -1
 
     #- Bright Galaxy Survey: priorities 2000-2997
     # ADM reserve 2998 for MWS_WD (ensuring a priority below Dark Survey targets, just in case)
@@ -231,6 +243,13 @@ priorities:
         MWS_MAIN_RED_NORTH:           SAME_AS_MWS_BROAD_NORTH
         MWS_MAIN_RED_SOUTH:           SAME_AS_MWS_BROAD_NORTH
 
+    # ADM secondary target priorities. Probably all have very low UNOBS
+    scnd_mask:
+        VETO:                         {UNOBS:  0, DONE: 0, OBS: 0, DONOTOBSERVE: 0}
+        BACKSTOP:                     {UNOBS: 10, DONE: 2, OBS: 1, DONOTOBSERVE: 0}
+        DR16Q:                        {UNOBS: 10, DONE: 2, OBS: 1, DONOTOBSERVE: 0}
+
+
 # ADM INITIAL number of observations (NUMOBS) for each target bit
 # ADM SAME_AS_XXX means to use the NUMOBS for bitname XXX
 # ADM -1 means that the concept of NUMOBS doesn't apply to this bit
@@ -271,7 +290,7 @@ numobs:
         NEAR_BRIGHT_OBJECT: -1
         BGS_ANY: -1
         MWS_ANY: -1
-        SECONDARY_ANY: -1
+        SCND_ANY: -1
 
     # ADM initial number of observations for targets in the Bright Galaxy Survey
     bgs_mask:
@@ -305,3 +324,8 @@ numobs:
         MWS_MAIN_RED_NORTH:           0
         MWS_MAIN_RED_SOUTH:           0
 
+    # ADM initial number of observations for secondary targets
+    scnd_mask:
+        VETO: 1
+        BACKSTOP: 1
+        DR16Q: 1

py/desitarget/geomask.py

@@ -1163,3 +1163,63 @@ def is_in_gal_box(objs, lbbox, radec=False):
           & (gal.b.value >= bmin) & (gal.b.value < bmax))
 
     return ii
+
+
+def radec_match_to(matchto, objs, sep=1., radec=False):
+    """Match objects to a catalog list on RA/Dec.
+
+    Parameters
+    ----------
+    matchto : :class:`~numpy.ndarray` or `list`
+        Coordinates to match TO. Must include the columns "RA" and "DEC".
+    objs : :class:`~numpy.ndarray` or `list`
+        Objects that will be matched to `matchto`. Must include "RA" and "DEC".
+    sep : :class:`float`, defaults to 1 arcsecond
+        The separation at which to match `objs` to `matchto` in ARCSECONDS.
+    radec : :class:`bool`, optional, defaults to ``False``
+        If ``True`` then `objs` and `matchto` are [RA, Dec] lists instead of
+        rec arrays.
+
+    Returns
+    -------
+    :class:`~numpy.ndarray` (of integers)
+        The indexes in `match2` for which `objs` matches `match2` at < `sep`.
+    :class:`~numpy.ndarray` (of integers)
+        The indexes in `objs` for which `objs` matches `match2` at < `sep`.
+
+    Notes
+    -----
+        - Sense is important, here. Every coordinate pair in `objs` is matched
+          to `matchto`, but NOT every coordinate pair in `matchto` is matched to
+          `objs`. `matchto` should be thought of as the parent catalog being
+          matched to, in that we are looking for all the instances where `objs`
+          has a match in `matchto`. The returned indexes thus can never be longer
+          than `objs`. Consider this example:
+
+          >>> mainra, maindec = [100], [30]
+          >>> ras, decs = [100, 100, 100], [30, 30, 30]
+          >>>
+          >>> radec_match_to([mainra, maindec], [ras, decs], radec=True)
+          >>> Out: (array([0, 0, 0]), array([0, 1, 2]))
+          >>>
+          >>> radec_match_to([ras, decs], [mainra, maindec], radec=True)
+          >>> Out: (array([0]), array([0]))
+
+        - Only returns the CLOSEST match within `sep` arcseconds.
+    """
+    if radec:
+        ram, decm = matchto
+        ra, dec = objs
+    else:
+        ram, decm = matchto["RA"], matchto["DEC"]
+        ra, dec = objs["RA"], objs["DEC"]
+
+    cmatchto = SkyCoord(ram*u.degree, decm*u.degree)
+    cobjs = SkyCoord(ra*u.degree, dec*u.degree)
+
+    idmatchto, d2d, _ = cobjs.match_to_catalog_sky(cmatchto)
+    idobjs = np.arange(len(cobjs))
+
+    ii = d2d < sep*u.arcsec
+
+    return idmatchto[ii], idobjs[ii]

py/desitarget/gfa.py

@@ -147,7 +147,7 @@ def gaia_gfas_from_sweep(filename, maglim=18.):
     return gfas
 
 
-def gaia_in_file(infile, maglim=18):
+def gaia_in_file(infile, maglim=18, mindec=-30., mingalb=10.):
     """Retrieve the Gaia objects from a HEALPixel-split Gaia file.
 
     Parameters
@@ -156,6 +156,11 @@ def gaia_in_file(infile, maglim=18):
         File name of a single Gaia "healpix" file.
     maglim : :class:`float`, optional, defaults to 18
         Magnitude limit for GFAs in Gaia G-band.
+    mindec : :class:`float`, optional, defaults to -30
+        Minimum declination (o) to include for output Gaia objects.
+    mingalb : :class:`float`, optional, defaults to 10
+        Closest latitude to Galactic plane for output Gaia objects
+        (e.g. send 10 to limit to areas beyond -10o <= b < 10o)"
 
     Returns
     -------
@@ -203,6 +208,13 @@ def gaia_in_file(infile, maglim=18):
     # ADM populate the BRICKID columns.
     gfas["BRICKID"] = bricks.brickid(gfas["RA"], gfas["DEC"])
 
+    # ADM limit by Dec first to speed transform to Galactic coordinates.
+    decgood = is_in_box(gfas, [0., 360., mindec, 90.])
+    gfas = gfas[decgood]
+    # ADM now limit to requesed Galactic latitude range.
+    bbad = is_in_gal_box(gfas, [0., 360., -mingalb, mingalb])
+    gfas = gfas[~bbad]
+
     return gfas
 
 
@@ -247,7 +259,8 @@ def all_gaia_in_tiles(maglim=18, numproc=4, allsky=False,
     # ADM the critical function to run on every file.
     def _get_gaia_gfas(fn):
         '''wrapper on gaia_in_file() given a file name'''
-        return gaia_in_file(fn, maglim=maglim)
+        return gaia_in_file(fn, maglim=maglim,
+                            mindec=mindec, mingalb=mingalb)
 
     # ADM this is just to count sweeps files in _update_status.
     nfile = np.zeros((), dtype='i8')
@@ -276,14 +289,6 @@ def _update_status(result):
 
     gfas = np.concatenate(gfas)
 
-    log.info('limit to Dec > {}o and |Gal b| > {}o...t = {:.1f} mins'
-             .format(mindec, mingalb, (time()-t0)/60.))
-    # ADM limit by Dec first to speed transform to Galactic coordinates.
-    decgood = is_in_box(gfas, [0., 360., mindec, 90.])
-    gfas = gfas[decgood]
-    # ADM limit to requesed Galactic latitude range.
-    bbad = is_in_gal_box(gfas, [0., 360., -mingalb, mingalb])
-    gfas = gfas[~bbad]
     log.info('Retrieved {} Gaia objects...t = {:.1f} mins'
              .format(len(gfas), (time()-t0)/60.))