Merge 0aceb2a into 1088c92

bin/select_secondary

@@ -43,7 +43,7 @@ hdr = fitsio.read_header(fns[0], 'SCND_TARG')
 surv = hdr["SURVEY"].rstrip()
 
 # ADM find the SCND_DIR environment variable, if it wasn't passed.
-scxdir = _get_scxdir(ns.scnddir)
+scxdir = _get_scxdir(ns.scnddir, survey=surv)
 # ADM and augment the scxdir if this is an SV set of primary files.
 if surv != 'main':
     scxdir = os.path.join(scxdir, surv)

bin/select_sv_targets

@@ -67,7 +67,7 @@ ap.add_argument("--nomaskbits", action='store_true',
 ap.add_argument("--writeall", action='store_true',
                 help="Default behavior is to split targets by bright/dark-time surveys. Send this to ALSO write a file of ALL targets")
 ap.add_argument("-nos", "--nosecondary", action='store_true',
-                help="Do NOT create TARGETID look-up files for secondary targets in $SCNDIR/outdata/priminfo-drversion-desitargetversion/$dest.fits (where $dest is the basename of dest)")
+                help="Do NOT create TARGETID look-up files for secondary targets in $SCNDIR/outdata/desitargetversion/priminfo-drversion-desitargetversion/$dest.fits (where $dest is the basename of dest)")
 ap.add_argument("--scnddir",
                 help="Base directory of secondary target files (e.g. '/project/projectdirs/desi/target/secondary' at NERSC). "+
                 "Defaults to SCND_DIR environment variable. Not needed if --nosecondary is sent.")
@@ -80,13 +80,14 @@ ns = ap.parse_args()
 # ADM build the list of command line arguments as
 # ADM bundlefiles potentially needs to know about them.
 extra = " --numproc {}".format(ns.numproc)
-if ns.tcnames is not None:
-    extra += " --tcnames {}".format(ns.tcnames)
 nsdict = vars(ns)
-for nskey in ["noresolve", "nomaskbits", "writeall",
-              "nosecondary", "nobackup", "nochecksum"]:
-    if nsdict[nskey]:
-        extra += " --{}".format(nskey)
+for nskey in ["tcnames", "noresolve", "nomaskbits", "writeall",
+              "nosecondary", "nobackup", "nochecksum", "scnddir"]:
+    if isinstance(nsdict[nskey], bool):
+        if nsdict[nskey]:
+            extra += " --{}".format(nskey)
+    elif nsdict[nskey] is not None:
+        extra += " --{} {}".format(nskey, nsdict[nskey])
 
 infiles = io.list_sweepfiles(ns.sweepdir)
 if ns.sweepdir2 is not None:
@@ -148,25 +149,29 @@ targets, infn = select_targets(
     tcnames=tcnames, survey=survey, backup=not(ns.nobackup),
     resolvetargs=not(ns.noresolve), mask=not(ns.nomaskbits), return_infiles=True
 )
-# ADM Set the list of infiles actually processed by select_targets() to
-# ADM None if we DON'T want to write their checksums to the output file.
-if ns.nochecksum:
-    shatab = None
-else:
-    shatab = get_checksums(infn, verbose=True)
 
 if ns.bundlefiles is None:
+    # ADM Set the list of infiles actually processed by select_targets() to
+    # ADM None if we DON'T want to write their checksums to the output file.
+    if ns.nochecksum:
+        shatab = None
+    else:
+        shatab = get_checksums(infn, verbose=True)
+
     # ADM only run secondary functions if --nosecondary was not passed.
     scndout = None
     if not ns.nosecondary and len(targets) > 0:
         from desitarget.secondary import _get_scxdir, match_secondary
         # ADM read secondary target directory.
-        scxdir = _get_scxdir(ns.scnddir)
+        scxdir = _get_scxdir(ns.scnddir, survey=survey)
         # ADM construct a label for the secondary file for TARGETID look-ups.
-        # ADM use RELEASE to determine the release string for the targets.
-        drint = np.max(targets['RELEASE']//1000)
+        try:
+            drint = int(ns.sweepdir.split("dr")[1][0])
+        except (ValueError, IndexError, AttributeError):
+            drint = "X"
         scndoutdn = "priminfo-dr{}-{}".format(drint, desitarget_version)
-        scndoutdn = os.path.join(scxdir, survey, "outdata", scndoutdn)
+        scndoutdn = os.path.join(scxdir, survey,
+                                 "outdata", desitarget_version, scndoutdn)
         if not os.path.exists(scndoutdn):
             log.info("making directory...{}".format(scndoutdn))
             os.makedirs(scndoutdn)
@@ -181,7 +186,7 @@ if ns.bundlefiles is None:
         scndout = os.path.join(scndoutdn, scndoutfn)
         log.info("writing files of primary matches to...{}".format(scndout))
         targets = match_secondary(targets, scxdir, scndout, sep=1.,
-                                  pix=pixlist, nside=ns.nside)
+                                  pix=pixlist, nside=ns.nside, swfiles=infn)
 
     if ns.mask:
         targets = mask_targets(targets, inmaskfile=ns.mask, nside=nside)

bin/select_targets

@@ -69,7 +69,7 @@ ap.add_argument("--nomaskbits", action='store_true',
 ap.add_argument("--writeall", action='store_true',
                 help="Default behavior is to split targets by bright/dark-time surveys. Send this to ALSO write a file of ALL targets")
 ap.add_argument("-nos", "--nosecondary", action='store_true',
-                help="Do NOT create TARGETID look-up files for secondary targets in $SCNDIR/outdata/priminfo-drversion-desitargetversion/$dest.fits (where $dest is the basename of dest)")
+                help="Do NOT create TARGETID look-up files for secondary targets in $SCNDIR/outdata/desitargetversion/priminfo-drversion-desitargetversion/$dest.fits (where $dest is the basename of dest)")
 ap.add_argument("--scnddir",
                 help="Base directory of secondary target files (e.g. '/project/projectdirs/desi/target/secondary' at NERSC). "+
                 "Defaults to SCND_DIR environment variable. Not needed if --nosecondary is sent.")
@@ -82,13 +82,14 @@ ns = ap.parse_args()
 # ADM build the list of command line arguments as
 # ADM bundlefiles potentially needs to know about them.
 extra = " --numproc {}".format(ns.numproc)
-if ns.tcnames is not None:
-    extra += " --tcnames {}".format(ns.tcnames)
 nsdict = vars(ns)
-for nskey in ["noresolve", "nomaskbits", "writeall",
-              "nosecondary", "nobackup", "nochecksum"]:
-    if nsdict[nskey]:
-        extra += " --{}".format(nskey)
+for nskey in ["tcnames", "noresolve", "nomaskbits", "writeall",
+              "nosecondary", "nobackup", "nochecksum", "scnddir"]:
+    if isinstance(nsdict[nskey], bool):
+        if nsdict[nskey]:
+            extra += " --{}".format(nskey)
+    elif nsdict[nskey] is not None:
+        extra += " --{} {}".format(nskey, nsdict[nskey])
 
 infiles = io.list_sweepfiles(ns.sweepdir)
 if ns.sweepdir2 is not None:
@@ -149,25 +150,29 @@ targets, infn = select_targets(
     tcnames=tcnames, survey='main', backup=not(ns.nobackup),
     resolvetargs=not(ns.noresolve), mask=not(ns.nomaskbits), return_infiles=True
 )
-# ADM Set the list of infiles actually processed by select_targets() to
-# ADM None if we DON'T want to write their checksums to the output file.
-if ns.nochecksum:
-    shatab = None
-else:
-    shatab = get_checksums(infn, verbose=True)
 
 if ns.bundlefiles is None:
+    # ADM Set the list of infiles actually processed by select_targets() to
+    # ADM None if we DON'T want to write their checksums to the output file.
+    if ns.nochecksum:
+        shatab = None
+    else:
+        shatab = get_checksums(infn, verbose=True)
+
     # ADM only run secondary functions if --nosecondary was not passed.
     scndout = None
     if not ns.nosecondary and len(targets) > 0:
         from desitarget.secondary import _get_scxdir, match_secondary
         # ADM read secondary target directory.
         scxdir = _get_scxdir(ns.scnddir)
         # ADM construct a label for the secondary file for TARGETID look-ups.
-        # ADM use RELEASE to determine the release string for the targets.
-        drint = np.max(targets['RELEASE']//1000)
+        try:
+            drint = int(ns.sweepdir.split("dr")[1][0])
+        except (ValueError, IndexError, AttributeError):
+            drint = "X"
         scndoutdn = "priminfo-dr{}-{}".format(drint, desitarget_version)
-        scndoutdn = os.path.join(scxdir, "outdata", scndoutdn)
+        scndoutdn = os.path.join(scxdir,
+                                 "outdata", desitarget_version, scndoutdn)
         if not os.path.exists(scndoutdn):
             log.info("making directory...{}".format(scndoutdn))
             os.makedirs(scndoutdn)
@@ -182,7 +187,7 @@ if ns.bundlefiles is None:
         scndout = os.path.join(scndoutdn, scndoutfn)
         log.info("writing files of primary matches to...{}".format(scndout))
         targets = match_secondary(targets, scxdir, scndout, sep=1.,
-                                  pix=pixlist, nside=ns.nside)
+                                  pix=pixlist, nside=ns.nside, swfiles=infn)
 
     if ns.mask:
         targets = mask_targets(targets, inmaskfile=ns.mask, nside=nside)

doc/changes.rst

@@ -5,6 +5,12 @@ desitarget Change Log
 0.47.1 (unreleased)
 -------------------
 
+* First run of secondaries with real target files [`PR #669`_]. Includes:
+    * General optimization, updating and debugging of the secondary code.
+    * Get `TARGETIDs` from the input sweeps, not just the actual targets.
+    * Add the first full bitmask for the SV1 secondary target files.
+    * Updates to the data model to better reflect the primary targets.
+* Clean-up minor style and doc issues from `PR #636`_ [`PR #668`_].
 * Updates and bug fixes for DR9 now SV is on-sky [`PR #665`_]. Includes:
     * Pass `MASKBITS` column forward for GFAs.
     * Bug fixes necessitated by target files having a second extension.
@@ -15,12 +21,12 @@ desitarget Change Log
 * Simplify stellar SV bits [`PR #636`_]:
     * Secondary bit requirement for main stellar SV program to 4 bits.
     * Primary bright science WDs use the old algorithmic selection.
-* Clean-up minor style and doc issues from `PR #636`_ [`PR #668`_].
 
 .. _`PR #636`: https://github.com/desihub/desitarget/pull/636
 .. _`PR #664`: https://github.com/desihub/desitarget/pull/664
 .. _`PR #665`: https://github.com/desihub/desitarget/pull/665
 .. _`PR #668`: https://github.com/desihub/desitarget/pull/668
+.. _`PR #669`: https://github.com/desihub/desitarget/pull/669
 
 0.47.0 (2020-12-10)
 -------------------

py/desitarget/io.py

@@ -864,7 +864,7 @@ def write_secondary(targdir, data, primhdr=None, scxdir=None, obscon=None,
         header of the output `filename`.
     obscon : :class:`str`, optional, defaults to `None`
         Can pass one of "DARK" or "BRIGHT". If passed, don't write the
-        full set of secondary target that do not match a primary,
+        full set of secondary targets that do not match a primary,
         rather only write targets appropriate for "DARK|GRAY" or
         "BRIGHT" observing conditions. The relevant `PRIORITY_INIT`
         and `NUMOBS_INIT` columns will be derived from
@@ -886,13 +886,13 @@ def write_secondary(targdir, data, primhdr=None, scxdir=None, obscon=None,
     Two sets of files are written:
         - The file of secondary targets that do not match a primary
           target is written to `targdir`. Such secondary targets
-          are determined from having `RELEASE==0` and `SKY==0`
-          in the `TARGETID`. Only targets with `PRIORITY_INIT > -1`
-          are written to this file (this allows duplicates to be
-          resolved in, e.g., :func:`~desitarget.secondary.finalize()`
+          are determined from having "PRIM_MATCH"=``False`` in `data`.
+          Only targets with `PRIORITY_INIT > -1` are written to this file
+          (this allows duplicates to be resolved in, e.g.,
+          :func:`~desitarget.secondary.finalize()`.
         - Each secondary target that, presumably, was initially drawn
           from the "indata" subdirectory of `scxdir` is written to
-          an "outdata/targdir" subdirectory of `scxdir`.
+          the "outdata" subdirectory of `scxdir`.
     """
     # ADM grab the scxdir, it it wasn't passed.
     from desitarget.secondary import _get_scxdir
@@ -924,9 +924,13 @@ def write_secondary(targdir, data, primhdr=None, scxdir=None, obscon=None,
         np.random.seed(616)
         data["SUBPRIORITY"] = np.random.random(ntargs)
 
-    # ADM remove the SCND_TARGET_INIT and SCND_ORDER columns.
-    scnd_target_init, scnd_order = data["SCND_TARGET_INIT"], data["SCND_ORDER"]
-    data = rfn.drop_fields(data, ["SCND_TARGET_INIT", "SCND_ORDER"])
+    # ADM remove the SCND_TARGET_INIT, SCND_ORDER and PRIM_MATCH columns.
+    scnd_target_init = data["SCND_TARGET_INIT"]
+    scnd_order = data["SCND_ORDER"]
+    prim_match = data["PRIM_MATCH"]
+
+    data = rfn.drop_fields(data,
+                           ["SCND_TARGET_INIT", "SCND_ORDER", "PRIM_MATCH"])
     # ADM we only need a subset of the columns where we match a primary.
     smalldata = rfn.drop_fields(data, ["PRIORITY_INIT", "SUBPRIORITY",
                                        "NUMOBS_INIT", "OBSCONDITIONS"])
@@ -937,14 +941,15 @@ def write_secondary(targdir, data, primhdr=None, scxdir=None, obscon=None,
     scnd_mask = mx[3]
 
     # ADM construct the output full and reduced file name.
-    filename = find_target_files(targdir, dr=drint, flavor="targets",
-                                 survey=survey, obscon=obscon, nohp=True)
-    filenam = os.path.splitext(os.path.basename(filename))[0]
+    filename = find_target_files(targdir, dr=drint, flavor="targets", nohp=True,
+                                 survey=survey, obscon=obscon, resolve=None)
 
     # ADM write out the file of matches for every secondary bit.
-    scxoutdir = os.path.join(scxdir, 'outdata', filenam)
+    scxoutdir = os.path.join(scxdir, 'outdata', desitarget_version)
     if obscon is not None:
         scxoutdir = os.path.join(scxoutdir, obscon.lower())
+    else:
+        scxoutdir = os.path.join(scxoutdir, "no-obscon")
     os.makedirs(scxoutdir, exist_ok=True)
 
     # ADM and write out the information for each bit.
@@ -968,9 +973,8 @@ def write_secondary(targdir, data, primhdr=None, scxdir=None, obscon=None,
     os.makedirs(os.path.dirname(filename), exist_ok=True)
 
     # ADM standalone secondaries have PRIORITY_INIT > -1 and
-    # ADM release before DR1 (release < 1000).
-    objid, brickid, release, mock, sky, gaiadr = decode_targetid(data["TARGETID"])
-    ii = (release < 1000) & (data["PRIORITY_INIT"] > -1)
+    # ADM don't have PRIM_MATCH set.
+    ii = ~prim_match & (data["PRIORITY_INIT"] > -1)
 
     # ADM ...write them out.
     write_with_units(filename, data[ii], extname='SCND_TARGETS', header=hdr)
@@ -2162,6 +2166,7 @@ def find_target_files(targdir, dr='X', flavor="targets", survey="main",
     resolve : :class:`bool`, optional, defaults to ``True``
         If ``True`` then find the `resolve` file. Otherwise find the
         `noresolve` file. Relevant if `flavor` is `targets` or `randoms`.
+        Pass ``None`` to substitute `resolve` with "secondary".
     supp : :class:`bool`, optional, defaults to ``False``
         If ``True`` then find the supplemental targets file. Overrides
         the `obscon` option.
@@ -2219,8 +2224,11 @@ def find_target_files(targdir, dr='X', flavor="targets", survey="main",
         log.critical(msg)
         raise ValueError(msg)
     res = "noresolve"
-    if resolve:
-        res = "resolve"
+    if resolve is None:
+        res = "secondary"
+    else:
+        if resolve:
+            res = "resolve"
     resdir = ""
     if flavor in ["targets", "randoms"]:
         resdir = res