First run of SV secondaries with real target files.

bin/select_secondary

@@ -43,7 +43,7 @@ hdr = fitsio.read_header(fns[0], 'SCND_TARG')
 surv = hdr["SURVEY"].rstrip()
 
 # ADM find the SCND_DIR environment variable, if it wasn't passed.
-scxdir = _get_scxdir(ns.scnddir)
+scxdir = _get_scxdir(ns.scnddir, survey=surv)
 # ADM and augment the scxdir if this is an SV set of primary files.
 if surv != 'main':
     scxdir = os.path.join(scxdir, surv)

bin/select_sv_targets

@@ -67,7 +67,7 @@ ap.add_argument("--nomaskbits", action='store_true',
 ap.add_argument("--writeall", action='store_true',
                 help="Default behavior is to split targets by bright/dark-time surveys. Send this to ALSO write a file of ALL targets")
 ap.add_argument("-nos", "--nosecondary", action='store_true',
-                help="Do NOT create TARGETID look-up files for secondary targets in $SCNDIR/outdata/priminfo-drversion-desitargetversion/$dest.fits (where $dest is the basename of dest)")
+                help="Do NOT create TARGETID look-up files for secondary targets in $SCNDIR/outdata/desitargetversion/priminfo-drversion-desitargetversion/$dest.fits (where $dest is the basename of dest)")
 ap.add_argument("--scnddir",
                 help="Base directory of secondary target files (e.g. '/project/projectdirs/desi/target/secondary' at NERSC). "+
                 "Defaults to SCND_DIR environment variable. Not needed if --nosecondary is sent.")
@@ -80,13 +80,14 @@ ns = ap.parse_args()
 # ADM build the list of command line arguments as
 # ADM bundlefiles potentially needs to know about them.
 extra = " --numproc {}".format(ns.numproc)
-if ns.tcnames is not None:
-    extra += " --tcnames {}".format(ns.tcnames)
 nsdict = vars(ns)
-for nskey in ["noresolve", "nomaskbits", "writeall",
-              "nosecondary", "nobackup", "nochecksum"]:
-    if nsdict[nskey]:
-        extra += " --{}".format(nskey)
+for nskey in ["tcnames", "noresolve", "nomaskbits", "writeall",
+              "nosecondary", "nobackup", "nochecksum", "scnddir"]:
+    if isinstance(nsdict[nskey], bool):
+        if nsdict[nskey]:
+            extra += " --{}".format(nskey)
+    elif nsdict[nskey] is not None:
+        extra += " --{} {}".format(nskey, nsdict[nskey])
 
 infiles = io.list_sweepfiles(ns.sweepdir)
 if ns.sweepdir2 is not None:
@@ -148,25 +149,29 @@ targets, infn = select_targets(
     tcnames=tcnames, survey=survey, backup=not(ns.nobackup),
     resolvetargs=not(ns.noresolve), mask=not(ns.nomaskbits), return_infiles=True
 )
-# ADM Set the list of infiles actually processed by select_targets() to
-# ADM None if we DON'T want to write their checksums to the output file.
-if ns.nochecksum:
-    shatab = None
-else:
-    shatab = get_checksums(infn, verbose=True)
 
 if ns.bundlefiles is None:
+    # ADM Set the list of infiles actually processed by select_targets() to
+    # ADM None if we DON'T want to write their checksums to the output file.
+    if ns.nochecksum:
+        shatab = None
+    else:
+        shatab = get_checksums(infn, verbose=True)
+
     # ADM only run secondary functions if --nosecondary was not passed.
     scndout = None
     if not ns.nosecondary and len(targets) > 0:
         from desitarget.secondary import _get_scxdir, match_secondary
         # ADM read secondary target directory.
-        scxdir = _get_scxdir(ns.scnddir)
+        scxdir = _get_scxdir(ns.scnddir, survey=survey)
         # ADM construct a label for the secondary file for TARGETID look-ups.
-        # ADM use RELEASE to determine the release string for the targets.
-        drint = np.max(targets['RELEASE']//1000)
+        try:
+            drint = int(ns.sweepdir.split("dr")[1][0])
+        except (ValueError, IndexError, AttributeError):
+            drint = "X"
         scndoutdn = "priminfo-dr{}-{}".format(drint, desitarget_version)
-        scndoutdn = os.path.join(scxdir, survey, "outdata", scndoutdn)
+        scndoutdn = os.path.join(scxdir, survey,
+                                 "outdata", desitarget_version, scndoutdn)
         if not os.path.exists(scndoutdn):
             log.info("making directory...{}".format(scndoutdn))
             os.makedirs(scndoutdn)
@@ -181,7 +186,7 @@ if ns.bundlefiles is None:
         scndout = os.path.join(scndoutdn, scndoutfn)
         log.info("writing files of primary matches to...{}".format(scndout))
         targets = match_secondary(targets, scxdir, scndout, sep=1.,
-                                  pix=pixlist, nside=ns.nside)
+                                  pix=pixlist, nside=ns.nside, swfiles=infn)
 
     if ns.mask:
         targets = mask_targets(targets, inmaskfile=ns.mask, nside=nside)

bin/select_targets

@@ -69,7 +69,7 @@ ap.add_argument("--nomaskbits", action='store_true',
 ap.add_argument("--writeall", action='store_true',
                 help="Default behavior is to split targets by bright/dark-time surveys. Send this to ALSO write a file of ALL targets")
 ap.add_argument("-nos", "--nosecondary", action='store_true',
-                help="Do NOT create TARGETID look-up files for secondary targets in $SCNDIR/outdata/priminfo-drversion-desitargetversion/$dest.fits (where $dest is the basename of dest)")
+                help="Do NOT create TARGETID look-up files for secondary targets in $SCNDIR/outdata/desitargetversion/priminfo-drversion-desitargetversion/$dest.fits (where $dest is the basename of dest)")
 ap.add_argument("--scnddir",
                 help="Base directory of secondary target files (e.g. '/project/projectdirs/desi/target/secondary' at NERSC). "+
                 "Defaults to SCND_DIR environment variable. Not needed if --nosecondary is sent.")
@@ -82,13 +82,14 @@ ns = ap.parse_args()
 # ADM build the list of command line arguments as
 # ADM bundlefiles potentially needs to know about them.
 extra = " --numproc {}".format(ns.numproc)
-if ns.tcnames is not None:
-    extra += " --tcnames {}".format(ns.tcnames)
 nsdict = vars(ns)
-for nskey in ["noresolve", "nomaskbits", "writeall",
-              "nosecondary", "nobackup", "nochecksum"]:
-    if nsdict[nskey]:
-        extra += " --{}".format(nskey)
+for nskey in ["tcnames", "noresolve", "nomaskbits", "writeall",
+              "nosecondary", "nobackup", "nochecksum", "scnddir"]:
+    if isinstance(nsdict[nskey], bool):
+        if nsdict[nskey]:
+            extra += " --{}".format(nskey)
+    elif nsdict[nskey] is not None:
+        extra += " --{} {}".format(nskey, nsdict[nskey])
 
 infiles = io.list_sweepfiles(ns.sweepdir)
 if ns.sweepdir2 is not None:
@@ -149,25 +150,29 @@ targets, infn = select_targets(
     tcnames=tcnames, survey='main', backup=not(ns.nobackup),
     resolvetargs=not(ns.noresolve), mask=not(ns.nomaskbits), return_infiles=True
 )
-# ADM Set the list of infiles actually processed by select_targets() to
-# ADM None if we DON'T want to write their checksums to the output file.
-if ns.nochecksum:
-    shatab = None
-else:
-    shatab = get_checksums(infn, verbose=True)
 
 if ns.bundlefiles is None:
+    # ADM Set the list of infiles actually processed by select_targets() to
+    # ADM None if we DON'T want to write their checksums to the output file.
+    if ns.nochecksum:
+        shatab = None
+    else:
+        shatab = get_checksums(infn, verbose=True)
+
     # ADM only run secondary functions if --nosecondary was not passed.
     scndout = None
     if not ns.nosecondary and len(targets) > 0:
         from desitarget.secondary import _get_scxdir, match_secondary
         # ADM read secondary target directory.
         scxdir = _get_scxdir(ns.scnddir)
         # ADM construct a label for the secondary file for TARGETID look-ups.
-        # ADM use RELEASE to determine the release string for the targets.
-        drint = np.max(targets['RELEASE']//1000)
+        try:
+            drint = int(ns.sweepdir.split("dr")[1][0])
+        except (ValueError, IndexError, AttributeError):
+            drint = "X"
         scndoutdn = "priminfo-dr{}-{}".format(drint, desitarget_version)
-        scndoutdn = os.path.join(scxdir, "outdata", scndoutdn)
+        scndoutdn = os.path.join(scxdir,
+                                 "outdata", desitarget_version, scndoutdn)
         if not os.path.exists(scndoutdn):
             log.info("making directory...{}".format(scndoutdn))
             os.makedirs(scndoutdn)
@@ -182,7 +187,7 @@ if ns.bundlefiles is None:
         scndout = os.path.join(scndoutdn, scndoutfn)
         log.info("writing files of primary matches to...{}".format(scndout))
         targets = match_secondary(targets, scxdir, scndout, sep=1.,
-                                  pix=pixlist, nside=ns.nside)
+                                  pix=pixlist, nside=ns.nside, swfiles=infn)
 
     if ns.mask:
         targets = mask_targets(targets, inmaskfile=ns.mask, nside=nside)

doc/changes.rst

@@ -5,6 +5,14 @@ desitarget Change Log
 0.47.1 (unreleased)
 -------------------
 
+* First run of secondaries with real target files [`PR #669`_]. Includes:
+    * Add Gaia-only standard stars to the MWS masks for SV, Main Survey:
+        * `GAIA_STD_FAINT`, `GAIA_STD_BRIGHT`, `GAIA_STD_WD`.
+    * General optimization, updating and debugging of the secondary code.
+    * Get `TARGETIDs` from the input sweeps, not just the actual targets.
+    * Add the first full bitmask for the SV1 secondary target files.
+    * Updates to the data model to better reflect the primary targets.
+* Clean-up minor style and doc issues from `PR #636`_ [`PR #668`_].
 * Updates and bug fixes for DR9 now SV is on-sky [`PR #665`_]. Includes:
     * Pass `MASKBITS` column forward for GFAs.
     * Bug fixes necessitated by target files having a second extension.
@@ -15,12 +23,12 @@ desitarget Change Log
 * Simplify stellar SV bits [`PR #636`_]:
     * Secondary bit requirement for main stellar SV program to 4 bits.
     * Primary bright science WDs use the old algorithmic selection.
-* Clean-up minor style and doc issues from `PR #636`_ [`PR #668`_].
 
 .. _`PR #636`: https://github.com/desihub/desitarget/pull/636
 .. _`PR #664`: https://github.com/desihub/desitarget/pull/664
 .. _`PR #665`: https://github.com/desihub/desitarget/pull/665
 .. _`PR #668`: https://github.com/desihub/desitarget/pull/668
+.. _`PR #669`: https://github.com/desihub/desitarget/pull/669
 
 0.47.0 (2020-12-10)
 -------------------