Merge pull request #678 from desihub/ADM-ded-sec

Finalize dedicated secondaries for SV1 to SV2 transition
desihub · Mar 7, 2021 · 879ad5e · 879ad5e
2 parents 8e5ac38 + 66a2d65
commit 879ad5e
Show file tree

Hide file tree

Showing 5 changed files with 175 additions and 133 deletions.
diff --git a/doc/changes.rst b/doc/changes.rst
@@ -5,6 +5,9 @@ desitarget Change Log
 0.50.1 (unreleased)
 -------------------
 
+* Finalize dedicated secondaries for sv1->sv2 transition [`PR #678`_].
+    * Deprecates Targets of Opportunity as standard secondary targets.
+    * ToOs will be handled by a separate ledger-based mechanism.
 * Main Survey, secondary, outside-footprint target updates [`PR #677`_]:
     * Update priorities for some secondary programs.
     * MTL changes to reobserve 0.7 < z < 2.1 QSOs at low priority.
@@ -26,6 +29,7 @@ desitarget Change Log
 .. _`PR #675`: https://github.com/desihub/desitarget/pull/675
 .. _`PR #676`: https://github.com/desihub/desitarget/pull/676
 .. _`PR #677`: https://github.com/desihub/desitarget/pull/677
+.. _`PR #678`: https://github.com/desihub/desitarget/pull/678
 
 
 0.50.0 (2021-01-29)

diff --git a/py/desitarget/data/targetmask.yaml b/py/desitarget/data/targetmask.yaml
@@ -125,8 +125,8 @@ scnd_mask:
        {obsconditions: DARK|GRAY, filename: 'dr16q', downsample: 1}]
 
 # ADM reserve 60/61 in scnd_mask for Targets of Opportunity in both SV and the Main Survey.
-    - [BRIGHT_TOO,    60, "Targets of Opportunity",   {obsconditions: BRIGHT,      filename: 'ToO'}]
-    - [DARK_TOO,      61, "Targets of Opportunity",   {obsconditions: DARK,        filename: 'ToO'}]
+    - [BRIGHT_TOO,    60, "Targets of Opportunity from rolling ledger",   {obsconditions: BRIGHT, flavor: 'TOO'}]
+    - [DARK_TOO,      61, "Targets of Opportunity from rolling ledger",   {obsconditions: DARK,   flavor: 'TOO'}]
 
 #- Observing conditions
 #- These are a bitmask to allow target bits to specify multiple conditions

diff --git a/py/desitarget/io.py b/py/desitarget/io.py
@@ -954,20 +954,22 @@ def write_secondary(targdir, data, primhdr=None, scxdir=None, obscon=None,
 
     # ADM and write out the information for each bit.
     for name in scnd_mask.names():
-        # ADM construct the output file name.
-        fn = "{}.fits".format(scnd_mask[name].filename)
-        scxfile = os.path.join(scxoutdir, fn)
-        # ADM retrieve just the data with this bit set.
-        ii = (scnd_target_init & scnd_mask[name]) != 0
-        # ADM only proceed to the write stage if there are targets.
-        if np.sum(ii) > 0:
-            # ADM to reorder to match the original input order.
-            order = np.argsort(scnd_order[ii])
-            # ADM write to file.
-            write_with_units(scxfile, smalldata[ii][order], extname='TARGETS',
-                             header=hdr)
-            log.info('Info for {} secondaries written to {}'
-                     .format(np.sum(ii), scxfile))
+        # ADM Targets of Opportunity are handled separately.
+        if scnd_mask[name].flavor != 'TOO':
+            # ADM construct the output file name.
+            fn = "{}.fits".format(scnd_mask[name].filename)
+            scxfile = os.path.join(scxoutdir, fn)
+            # ADM retrieve just the data with this bit set.
+            ii = (scnd_target_init & scnd_mask[name]) != 0
+            # ADM only proceed to the write stage if there are targets.
+            if np.sum(ii) > 0:
+                # ADM to reorder to match the original input order.
+                order = np.argsort(scnd_order[ii])
+                # ADM write to file.
+                write_with_units(scxfile, smalldata[ii][order],
+                                 extname='TARGETS', header=hdr)
+                log.info('Info for {} secondaries written to {}'
+                         .format(np.sum(ii), scxfile))
 
     # ADM make necessary directories for the file, if they don't exist.
     os.makedirs(os.path.dirname(filename), exist_ok=True)

diff --git a/py/desitarget/secondary.py b/py/desitarget/secondary.py
@@ -241,7 +241,8 @@ def _check_files(scxdir, scnd_mask):
         setdic[subdir] = set(fns)
 
     # ADM check for bit correspondence.
-    setbitfns = set([scnd_mask[name].filename for name in scnd_mask.names()])
+    setbitfns = set([scnd_mask[name].filename for name in scnd_mask.names()
+                     if scnd_mask[name].flavor != "TOO"])
     if setbitfns != setdic['indata']:
         msg = "files in yaml file don't match files in {}\n".format(
             dirdic['indata'])
@@ -294,75 +295,78 @@ def read_files(scxdir, scnd_mask):
     scxall = []
     # ADM loop through all of the scx bits.
     for name in scnd_mask.names():
-        log.debug('SCND target: {}'.format(name))
-        # ADM the full file path without the extension.
-        fn = os.path.join(fulldir, scnd_mask[name].filename)
-        log.debug('     path:   {}'.format(fn))
-        # ADM if the relevant file is a .txt file, read it in.
-        if os.path.exists(fn+'.txt'):
-            try:
-                scxin = np.loadtxt(fn+'.txt', usecols=[0, 1, 2, 3, 4, 5],
-                                   dtype=indatamodel.dtype)
-            except (ValueError, IndexError):
-                msg = "First 6 columns don't correspond to {} in {}.txt".format(
-                    indatamodel.dtype, fn)
-                # ADM perhaps people provided .csv files as .txt files.
+        # ADM Targets of Opportunity are handled separately.
+        if scnd_mask[name].flavor != 'TOO':
+            log.debug('SCND target: {}'.format(name))
+            # ADM the full file path without the extension.
+            fn = os.path.join(fulldir, scnd_mask[name].filename)
+            log.debug('     path:   {}'.format(fn))
+            # ADM if the relevant file is a .txt file, read it in.
+            if os.path.exists(fn+'.txt'):
                 try:
                     scxin = np.loadtxt(fn+'.txt', usecols=[0, 1, 2, 3, 4, 5],
-                                       dtype=indatamodel.dtype, delimiter=",")
+                                       dtype=indatamodel.dtype)
                 except (ValueError, IndexError):
-                    log.error(msg)
-                    raise IOError(msg)
-
-        # ADM otherwise it's a fits file, read it in.
-        else:
-            scxin = fitsio.read(fn+'.fits',
-                                columns=indatamodel.dtype.names)
-
-        # ADM ensure this is a properly constructed numpy array.
-        scxin = np.atleast_1d(scxin)
-
-        # ADM assert the data model.
-        msg = "Data model doesn't match {} in {}".format(indatamodel.dtype, fn)
-        for col in indatamodel.dtype.names:
-            assert scxin[col].dtype == indatamodel[col].dtype, msg
-
-        # ADM check RA/Dec are reasonable.
-        outofbounds = ((scxin["RA"] >= 360.) | (scxin["RA"] < 0) |
-                       (scxin["DEC"] > 90) | (scxin["DEC"] < -90))
-        if np.any(outofbounds):
-            msg = "RA/Dec outside of range in {}; RA={}, Dec={}".format(
-                fn, scxin["RA"][outofbounds], scxin["DEC"][outofbounds])
-            log.error(msg)
-            raise IOError(msg)
-
-        # ADM now checks are done, downsample to the required density.
-        log.debug("Read {} targets from {}".format(len(scxin), fn))
-        ds = scnd_mask[name].downsample
-        if ds < 1:
-            log.debug("Downsampling to first {}% of file".format(100*ds))
-        scxin = scxin[:int(len(scxin)*ds)]
-        log.debug("Working with {} targets for {}".format(len(scxin), name))
-
-        # ADM the default is 2015.5 for the REF_EPOCH.
-        ii = scxin["REF_EPOCH"] == 0
-        scxin["REF_EPOCH"][ii] = 2015.5
-
-        # ADM add the other output columns.
-        dt = outdatamodel.dtype.descr + suppdatamodel.dtype.descr
-        scxout = np.zeros(len(scxin), dtype=dt)
-        for col in indatamodel.dtype.names:
-            scxout[col] = scxin[col]
-        scxout["SCND_TARGET"] = scnd_mask[name]
-        scxout["SCND_TARGET_INIT"] = scnd_mask[name]
-        scxout["SCND_ORDER"] = np.arange(len(scxin))
-        scxout["PRIORITY_INIT"] = scnd_mask[name].priorities['UNOBS']
-        scxout["NUMOBS_INIT"] = scnd_mask[name].numobs
-        scxout["TARGETID"] = -1
-        scxout["OBSCONDITIONS"] =     \
-            obsconditions.mask(scnd_mask[name].obsconditions)
-        scxout["PRIM_MATCH"] = False
-        scxall.append(scxout)
+                    msg = "First 6 columns don't look like {} in {}.txt".format(
+                        indatamodel.dtype, fn)
+                    # ADM perhaps people provided .csv files as .txt files.
+                    try:
+                        scxin = np.loadtxt(
+                            fn+'.txt', usecols=[0, 1, 2, 3, 4, 5],
+                            dtype=indatamodel.dtype, delimiter=",")
+                    except (ValueError, IndexError):
+                        log.error(msg)
+                        raise IOError(msg)
+
+            # ADM otherwise it's a fits file, read it in.
+            else:
+                scxin = fitsio.read(fn+'.fits',
+                                    columns=indatamodel.dtype.names)
+
+            # ADM ensure this is a properly constructed numpy array.
+            scxin = np.atleast_1d(scxin)
+
+            # ADM assert the data model.
+            msg = "Data model mismatch: {} in {}".format(indatamodel.dtype, fn)
+            for col in indatamodel.dtype.names:
+                assert scxin[col].dtype == indatamodel[col].dtype, msg
+
+            # ADM check RA/Dec are reasonable.
+            outofbounds = ((scxin["RA"] >= 360.) | (scxin["RA"] < 0) |
+                           (scxin["DEC"] > 90) | (scxin["DEC"] < -90))
+            if np.any(outofbounds):
+                msg = "RA/Dec outside of range in {}; RA={}, Dec={}".format(
+                    fn, scxin["RA"][outofbounds], scxin["DEC"][outofbounds])
+                log.error(msg)
+                raise IOError(msg)
+
+            # ADM now checks are done, downsample to the required density.
+            log.debug("Read {} targets from {}".format(len(scxin), fn))
+            ds = scnd_mask[name].downsample
+            if ds < 1:
+                log.debug("Downsampling to first {}% of file".format(100*ds))
+            scxin = scxin[:int(len(scxin)*ds)]
+            log.debug("Working with {} targets for {}".format(len(scxin), name))
+
+            # ADM the default is 2015.5 for the REF_EPOCH.
+            ii = scxin["REF_EPOCH"] == 0
+            scxin["REF_EPOCH"][ii] = 2015.5
+
+            # ADM add the other output columns.
+            dt = outdatamodel.dtype.descr + suppdatamodel.dtype.descr
+            scxout = np.zeros(len(scxin), dtype=dt)
+            for col in indatamodel.dtype.names:
+                scxout[col] = scxin[col]
+            scxout["SCND_TARGET"] = scnd_mask[name]
+            scxout["SCND_TARGET_INIT"] = scnd_mask[name]
+            scxout["SCND_ORDER"] = np.arange(len(scxin))
+            scxout["PRIORITY_INIT"] = scnd_mask[name].priorities['UNOBS']
+            scxout["NUMOBS_INIT"] = scnd_mask[name].numobs
+            scxout["TARGETID"] = -1
+            scxout["OBSCONDITIONS"] =     \
+                obsconditions.mask(scnd_mask[name].obsconditions)
+            scxout["PRIM_MATCH"] = False
+            scxall.append(scxout)
 
     return np.concatenate(scxall)