Implement the full loop for MTL

bin/run_mtl_loop

@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+
+from desitarget.mtl import loop_ledger
+from desiutil.log import get_logger
+log = get_logger()
+
+# ADM default survey to run.
+survey = "main"
+
+from argparse import ArgumentParser
+ap = ArgumentParser(description='Make an initial HEALPixel-split ledger for a Merged Target List based on a directory of targets')
+ap.add_argument("obscon",
+                help="String matching ONE obscondition in the bitmask yaml file \
+                (e.g. 'BRIGHT'). Controls priorities when merging targets,      \
+                which tiles to process, etc.")
+ap.add_argument("-s", "--survey",
+                help="Flavor of survey to run. Defaults to [{}]".format(survey),
+                default=survey)
+ap.add_argument('--zcatdir',
+                help="Full path to the directory that hosts the redshift        \
+                catalogs. Default is to use the $ZCAT_DIR environment variable",
+                default=None)
+ap.add_argument('--mtldir',
+                help="Full path to the directory that hosts the MTL ledgers.    \
+                Default is to use the $ZCAT_DIR environment variable",
+                default=None)
+ap.add_argument('--tilefn',
+                help="Full path to the name of the tile file.                   \
+                Default is to use the $TILE_FN environment variable",
+                default=None)
+ap.add_argument("-n", "--numobsfromzcat", action='store_true',
+                help="If passed, the zcat includes a meaningfule NUMOBS.        \
+                Otherwise, NUMOBS is looked up from the ledger itself")
+
+ns = ap.parse_args()
+
+numobs_from_ledger = not(ns.numobsfromzcat)
+
+hpdirname, mtltilefn, tilefn, tiles = loop_ledger(
+    ns.obscon, survey=ns.survey, zcatdir=ns.zcatdir, mtldir=ns.mtldir,
+    tilefn=ns.tilefn, numobs_from_ledger=numobs_from_ledger)
+
+log.info("MTL ledger directory: {}".format(hpdirname))
+log.info("MTL tile file: {}".format(mtltilefn))
+log.info("Tile file: {}".format(tilefn))
+log.info("Processed {} new tiles, which are: {}".format(len(tiles), tiles))

doc/changes.rst

@@ -10,6 +10,15 @@ desitarget Change Log
 0.52.0 (2021-03-13)
 -------------------
 
+* Implement full MTL loop [`PR #684`_]. Includes:
+    * Modify ledgers based on any new tiles in a ``zcat`` directory.
+    * An MTL tile file to track which tiles have been processed by MTL.
+    * Read standard tile file to get observing conditions for each tile.
+        * Only update ledgers for tiles with the appropriate conditions.
+    * Option to use the ledgers themselves to updated ``NUMOBS``
+        * instead of expecting ``NUMOBS`` to be in the ``zcat``.
+    * A command-line script to execute the full loop.
+        * Input directories and files can be environment variables.
 * New secondary bits for COSMOS/unusual point sources [`PR #682`_].
 * Add formalism to make ledger for BACKUP targets [`PR #681`_].
 * New QSO target selection in SV2 [`PR #680`_] for validation:
@@ -31,6 +40,7 @@ desitarget Change Log
 .. _`PR #680`: https://github.com/desihub/desitarget/pull/680
 .. _`PR #681`: https://github.com/desihub/desitarget/pull/681
 .. _`PR #682`: https://github.com/desihub/desitarget/pull/682
+.. _`PR #684`: https://github.com/desihub/desitarget/pull/684
 
 
 0.51.0 (2021-03-07)

py/desitarget/data/targetmask.yaml

@@ -253,9 +253,9 @@ priorities:
         MWS_MAIN_RED:                 SAME_AS_MWS_MAIN_BLUE
         MWS_MAIN_RED_NORTH:           SAME_AS_MWS_BROAD_NORTH
         MWS_MAIN_RED_SOUTH:           SAME_AS_MWS_BROAD_NORTH
-        BACKUP_BRIGHT:                {UNOBS: 9, DONE: 9, OBS: 9, DONOTOBSERVE: 0}
-        BACKUP_FAINT:                 {UNOBS: 8, DONE: 8, OBS: 8, DONOTOBSERVE: 0}
-        BACKUP_VERY_FAINT:            {UNOBS: 7, DONE: 7, OBS: 7, DONOTOBSERVE: 0}
+        BACKUP_BRIGHT:                {UNOBS: 9, DONE: 2, OBS: 1, DONOTOBSERVE: 0}
+        BACKUP_FAINT:                 {UNOBS: 8, DONE: 2, OBS: 1, DONOTOBSERVE: 0}
+        BACKUP_VERY_FAINT:            {UNOBS: 7, DONE: 2, OBS: 1, DONOTOBSERVE: 0}
         # ADM Standards are special; priorities don't apply.
         GAIA_STD_FAINT:  -1
         GAIA_STD_WD:  -1

py/desitarget/data/units.yaml

@@ -177,6 +177,8 @@ TIMESTAMP: s
 VERSION:
 Z:
 ZWARN:
+ZTILEID:
+TILEID:
 
 # ADM ToO-related units:
 CHECKER:

py/desitarget/io.py

@@ -12,6 +12,7 @@
 # import pandas as pd
 import fitsio
 from astropy.table import Table
+from astropy.io import ascii
 import os
 import re
 from . import __version__ as desitarget_version
@@ -430,8 +431,10 @@ def write_with_units(filename, data, extname=None, header=None, ecsv=False):
 
     # ADM write the file for either ecsv or fits..
     if ecsv:
-        data.meta = dict(header)
-        data.meta['EXTNAME'] = extname
+        if header is not None:
+            data.meta = dict(header)
+        if extname is not None:
+            data.meta['EXTNAME'] = extname
         data.write(filename+'.tmp', format='ascii.ecsv', overwrite=True)
     else:
         fitsio.write(filename+'.tmp', data, units=units, extname=extname,
@@ -702,7 +705,7 @@ def write_mtl(mtldir, data, indir=None, survey="main", obscon=None,
         Written to output file header as the keyword `SURVEY`.
     obscon : :class:`str`, optional
         Name of the `OBSCONDITIONS` used to make the file (e.g. DARK).
-    nsidefile : :class:`int`, optional, defaults to `mtl.get_mtl_dir()`
+    nsidefile : :class:`int`, optional
         Passed to indicate in the output file header that the targets
         have been limited to only certain HEALPixels at a given
         nside. Used in conjunction with `hpxlist`.
@@ -2314,6 +2317,8 @@ def find_target_files(targdir, dr='X', flavor="targets", survey="main",
 
     # ADM the generic directory structure beneath $TARG_DIR or $MTL_DIR.
     fn = os.path.join(targdir, drstr, version, flavor)
+    if flavor == "mtl":
+        fn = targdir
 
     # ADM masks are a special case beneath $MASK_DIR.
     if flavor == "masks":
@@ -2365,6 +2370,67 @@ def find_target_files(targdir, dr='X', flavor="targets", survey="main",
     return fn
 
 
+def read_mtl_tile_file(filename):
+    """Read which tiles have been processed by MTL from the tile file.
+
+    Parameters
+    ----------
+    filename : :class:`str`
+        The full path to the MTL tile file.
+
+    Returns
+    -------
+    :class:`~numpy.ndarray`
+        A structured numpy array of the MTL tile file.
+    """
+    from desitarget.mtl import mtltilefiledm as dm
+    dt = dm.dtype
+    mtltiles = Table.read(filename, comment="#", delimiter=" ",
+                          format='pandas.csv', dtype=dt)
+
+    return mtltiles
+
+
+def write_mtl_tile_file(filename, data):
+    """Append new tiles to the end of the MTL tile file.
+
+    Parameters
+    ----------
+    filename : :class:`str`
+        The full path to the MTL tile file. If this doesn't exist, then
+        a new file is begun.
+    data : :class`~numpy.ndarray`
+        The data to append to the end of the MTL tile file, which should
+        be in the format of desitarget.mtl.mtltilefiledm
+
+    Returns
+    -------
+    :class:`int`
+        The number of targets that were written to file.
+    :class:`str`
+        The name of the file to which targets were written.
+    """
+    from desitarget.mtl import mtltilefiledm as dm
+    dt = dm.dtype
+    if data.dtype != dt:
+        msg = "Data for MTL tile file formatted as {}. Should be {}".format(
+            data.dtype, dt)
+        log.error(msg)
+        raise IOError(msg)
+
+    if os.path.isfile(filename):
+        # ADM as we're working with .ecsv, simply append to the end.
+        f = open(filename, "a")
+        ascii.write(data, f, format='no_header')
+        f.close()
+    else:
+        # ADM if the file doesn't exist we may need to make the directory.
+        os.makedirs(os.path.dirname(filename), exist_ok=True)
+        write_with_units(filename, data, extname='MTLTILE', ecsv=True)
+
+    return len(data), filename
+
+
 def read_mtl_ledger(filename, unique=True):
     """Wrapper to read individual MTL ledger files.