Merge pull request #739 from desihub/ADM-hack-init-MTL

Allow initial ledgers to use a preordained timestamp

bin/make_initial_mtl_ledger

@@ -37,6 +37,14 @@ ap.add_argument('--healpixels',
                 write the ledger for targets in these pixels, at the default    \
                 nside, which is [{}]".format(mtlnside),
                 default=None)
+ap.add_argument('--timestamp',
+                help="Override the TIMESTAMP created by make_mtl, and use this  \
+                time instead",
+                default=None)
+ap.add_argument('--exemptmf',
+                help="Exempt targets that are purely MWS_FAINT targets from the \
+                TIMESTAMP override",
+                action='store_true')
 
 ns = ap.parse_args()
 
@@ -45,4 +53,5 @@ pixlist = ns.healpixels
 if pixlist is not None:
     pixlist = [int(pix) for pix in pixlist.split(',')]
 
-make_ledger(ns.targdir, ns.dest, pixlist=pixlist, obscon=ns.obscon, numproc=ns.numproc)
+make_ledger(ns.targdir, ns.dest, pixlist=pixlist, obscon=ns.obscon,
+            numproc=ns.numproc, timestamp=ns.timestamp, exemptmf=ns.exemptmf)

doc/changes.rst

@@ -4,7 +4,12 @@ desitarget Change Log
 
 1.0.2 (unreleased)
 ------------------
-
+* Allow initial ledgers to use a preordained timestamp [`PR #739`_].
+    * ``MWS_FAINT`` targets can be exempted from this timestamp.
+    * Also change data model for initial ledgers:
+        * ``ZS`` and ``ZINFO`` are replaced by ``Z_QN``
+	* This is not backwards-compatible to version `1.0.0`
+    * Fix `PR #734`_ bug where ``hpxlist`` was used in `write_secondary`.
 * Ensure fixed order of input files for reproducible outputs [`PR #738`_].
 * Refactor data model and I/O for the QSO zcats [`PR #737`_]. Includes:
     * New directory structures.
@@ -26,6 +31,7 @@ desitarget Change Log
 .. _`PR #734`: https://github.com/desihub/desitarget/pull/734
 .. _`PR #737`: https://github.com/desihub/desitarget/pull/737
 .. _`PR #738`: https://github.com/desihub/desitarget/pull/738
+.. _`PR #739`: https://github.com/desihub/desitarget/pull/739
 
 1.0.1 (2021-05-14)
 ------------------

py/desitarget/io.py

@@ -934,8 +934,7 @@ def write_secondary(targdir, data, primhdr=None, scxdir=None, obscon=None,
     subpriority : :class:`bool`, optional, defaults to ``True``
         If ``True`` and a `SUBPRIORITY` column is in the input `data`,
         then `SUBPRIORITY` is overwritten by a random float in the range
-        0 to 1, using either seed 717, or seed 717 + the first value in
-        `hpxlist`, if `hpxlist` is passed and not ``None``.
+        0 to 1, using seed 717.
 
     Returns
     -------
@@ -991,8 +990,6 @@ def write_secondary(targdir, data, primhdr=None, scxdir=None, obscon=None,
     if "SUBPRIORITY" in data.dtype.names and subpriority:
         ntargs = len(data)
         subpseed = 717
-        if hpxlist is not None:
-            subpseed += int(hpxlist[0])
         np.random.seed(subpseed)
         data["SUBPRIORITY"] = np.random.random(ntargs)
         hdr["SUBPSEED"] = subpseed

py/desitarget/mtl.py

@@ -49,7 +49,7 @@
 
 # ADM columns to add to the mtl/zcat data models for the Main Survey.
 msaddcols = np.array([], dtype=[
-    ('ZS', 'U2'), ('ZINFO', 'U8'), ('DELTACHI2', '>f8'),
+    ('Z_QN', '>f8'), ('DELTACHI2', '>f8'),
     ])
 
 zcatdatamodel = np.array([], dtype=[
@@ -99,6 +99,41 @@ def survey_data_model(dm, survey='main'):
         raise ValueError(msg)
 
 
+def check_timestamp(timestamp):
+    """Check whether a timestamp is in a valid datetime format.
+
+    Parameters
+    ----------
+    date : :class:`str`
+        A string that should be a valid datetime string with a timezone.
+
+    Returns
+    -------
+    :class:`str`
+        The input `date` string.
+
+    Notes
+    -----
+    - Triggers an exception if the string is not a valid datetime string
+      or if the timezone was not included in the string.
+    """
+    from dateutil.parser import parse
+
+    try:
+        check = parse(timestamp)
+    except ValueError:
+        msg = "{} is not a valid timestamp!!!".format(timestamp)
+        log.critical(msg)
+        raise ValueError(msg)
+
+    if check.tzinfo is None:
+        msg = "{} does not include timezone information!!!".format(timestamp)
+        log.critical(msg)
+        raise ValueError(msg)
+
+    return timestamp
+
+
 def get_utc_date(survey="sv3"):
     """Convenience function to grab the UTC date.
 
@@ -530,7 +565,8 @@ def make_mtl(targets, obscon, zcat=None, scnd=None,
 
 
 def make_ledger_in_hp(targets, outdirname, nside, pixlist, obscon="DARK",
-                      indirname=None, verbose=True, scnd=False):
+                      indirname=None, verbose=True, scnd=False,
+                      timestamp=None, exemptmf=False):
     """
     Make an initial MTL ledger file for targets in a set of HEALPixels.
 
@@ -557,20 +593,48 @@ def make_ledger_in_hp(targets, outdirname, nside, pixlist, obscon="DARK",
         If ``True`` then log target and file information.
     scnd : :class:`bool`, defaults to ``False``
         If ``True`` then this is a ledger of secondary targets.
+    timestamp : :class:`str`, optional
+        A timestamp to use in place of that assigned by `make_mtl`.
+    exemptmf : :class:`bool`, optional, defaults to ``False``
+        If ``True`` then exempt any target that has a `TARGET_STATE`
+        driven by `MWS_FAINT_*` classes from accepting `timestamp`. These
+        targets will instead revert to a TIMESTAMP corresponding to when
+        the code was run. This is to fix a bug where "MWS_FAINT" targets
+        were not included in the (1.0.0) Main Survey target files.
 
     Returns
     -------
     Nothing, but writes the `targets` out to `outdirname` split across
     each HEALPixel in `pixlist`.
     """
     t0 = time()
+    # ADM a dictionary to hold header keywords for the ouput file.
+    hdr = {}
 
     # ADM in case an integer was passed.
     pixlist = np.atleast_1d(pixlist)
 
     # ADM execute MTL.
     mtl = make_mtl(targets, obscon, trimcols=True)
 
+    # ADM if requested, substitute a bespoke timestamp.
+    if timestamp is not None:
+        hdr["TSFORCED"] = timestamp
+        hdr["EXEMPTMF"] = exemptmf
+        # ADM check the timestamp is valid.
+        _ = check_timestamp(timestamp)
+        origts = mtl["TIMESTAMP"].copy()
+        mtl["TIMESTAMP"] = timestamp
+        # ADM don't use the bespoke timestamp for MWS_FAINT_* targets.
+        if exemptmf:
+            # ADM do not update the timestamp for targets for which
+            # ADM MWS_FAINT_* is controlling the priority. As MWS_FAINT_*
+            # ADM targets are the lowest-priority UNOBSERVED targets,
+            # ADM this is equivalent to not updating the timestamp for
+            # ADM targets that are purely MWS_FAINT.
+            ii = np.array(["MWS_FAINT" in ts for ts in mtl["TARGET_STATE"]])
+            mtl["TIMESTAMP"][ii] = origts[ii]
+
     # ADM the HEALPixel within which each target in the MTL lies.
     theta, phi = np.radians(90-mtl["DEC"]), np.radians(mtl["RA"])
     mtlpix = hp.ang2pix(nside, theta, phi, nest=True)
@@ -584,15 +648,16 @@ def make_ledger_in_hp(targets, outdirname, nside, pixlist, obscon="DARK",
             nt, fn = io.write_mtl(
                 outdirname, mtl[inpix].as_array(), indir=indirname, ecsv=ecsv,
                 survey=survey, obscon=obscon, nsidefile=nside, hpxlist=pix,
-                scnd=scnd)
+                scnd=scnd, extra=hdr)
             if verbose:
                 log.info('{} targets written to {}...t={:.1f}s'.format(
                     nt, fn, time()-t0))
 
     return
 
 
-def make_ledger(hpdirname, outdirname, pixlist=None, obscon="DARK", numproc=1):
+def make_ledger(hpdirname, outdirname, pixlist=None, obscon="DARK",
+                numproc=1, timestamp=None, exemptmf=False):
     """
     Make initial MTL ledger files for HEALPixels, in parallel.
 
@@ -617,6 +682,14 @@ def make_ledger(hpdirname, outdirname, pixlist=None, obscon="DARK", numproc=1):
         governs the sub-directory to which the ledger is written.
     numproc : :class:`int`, optional, defaults to 1 for serial
         Number of processes to parallelize across.
+    timestamp : :class:`str`, optional
+        A timestamp to use in place of that assigned by `make_mtl`.
+    exemptmf : :class:`bool`, optional, defaults to ``False``
+        If ``True`` then exempt any target that has a `TARGET_STATE`
+        driven by `MWS_FAINT_*` classes from accepting `timestamp`. These
+        targets will instead revert to a TIMESTAMP corresponding to when
+        the code was run. This is to fix a bug where "MWS_FAINT" targets
+        were not included in the (1.0.0) Main Survey target files.
 
     Returns
     -------
@@ -703,7 +776,8 @@ def _make_ledger_in_hp(pixnum):
         # ADM write MTLs for the targs split over HEALPixels in pixlist.
         return make_ledger_in_hp(
             targs, outdirname, mtlnside, pix, obscon=obscon,
-            indirname=hpdirname, verbose=False, scnd=scnd)
+            indirname=hpdirname, verbose=False, scnd=scnd,
+            timestamp=timestamp, exemptmf=exemptmf)
 
     # ADM this is just to count pixels in _update_status.
     npix = np.ones((), dtype='i8')