Merge 482e39b into e434e51

bin/make_initial_mtl_ledger

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+
+from desitarget.mtl import make_ledger, get_mtl_dir
+#import warnings
+#warnings.simplefilter('error')
+
+# ADM this is the upper-limit for memory for mtl._get_mtl_nside()=32.
+nproc = 12
+# ADM this is the upper-limit for memory for mtl._get_mtl_nside()=16.
+# nproc = 8
+# ADM retrieve the $MTL_DIR environment variable.
+mtldir = get_mtl_dir()
+# ADM default obsconditions.
+obscon = "DARK"
+
+from argparse import ArgumentParser
+ap = ArgumentParser(description='Make an initial HEALPixel-split ledger for a Merged Target List based on a directory of targets')
+ap.add_argument("targdir",
+                help="Full path to a directory containing files of targets that \
+                have been partitioned by HEALPixel (i.e. as made by             \
+                `select_targets` with the `bundle_files` option).")
+ap.add_argument("--dest",
+                help="Full path to the output directory to host the ledger. The \
+                filename and full directory structure is built on-the-fly from  \
+                file headers in targdirname. [defaults to {})".format(mtldir),
+                default=mtldir)
+ap.add_argument("--obscon",
+                help="String matching ONE obscondition in the bitmask yaml file \
+                (e.g. 'GRAY'). Controls priorities when merging targets and     \
+                where the ledger is written. [defaults to {})".format(obscon),
+                default=obscon)
+ap.add_argument("--numproc", type=int,
+                help='number of concurrent processes to use [defaults to {}]'.
+                format(nproc),
+                default=nproc)
+
+ns = ap.parse_args()
+
+make_ledger(ns.targdir, ns.dest, obscon=ns.obscon, numproc=ns.numproc)

doc/changes.rst

@@ -5,7 +5,26 @@ desitarget Change Log
 0.41.1 (unreleased)
 -------------------
 
-* No changes yet.
+* Major refactor to MTL to implement ledgers [`PR #635`_]. Includes:
+    * Code to make initial HEALPix-split ledger files from target files.
+        * Ledgers can be produced for each observing layer.
+        * Also includes an easy-to-use binary executable script.
+        * New data model with timestamp, code version and target states.
+    * Code to rapidly update MTL information by appending to a ledger.
+        * Uses targets and a zcat with the current standard columns.
+    * Functionality that works with either FITS or ECSV files.
+    * Automatic trimming of target columns in :func:`mtl.make_mtl()`.
+        * Saves memory, which may help with processing of mocks.
+    * :func:`mtl.inflate_ledger()` to re-obtain trimmed target columns.
+    * Code to write MTL files in a standard format.
+    * Utility functions to read (FITS or ECSV) MTL ledgers:
+        * In a set of HEALPixels (:func:`io.read_mtl_in_hp`)
+        * In a set of tiles (:func:`read_targets_in_tiles` with mtl=True)
+        * In a box (:func:`read_targets_in_box` with mtl=True)
+        * In a cap (:func:`read_targets_in_cap` with mtl=True)
+    * Can read entire ledger, or most recent entry for each ``TARGETID``.
+
+.. _`PR #635`: https://github.com/desihub/desitarget/pull/635
 
 0.41.0 (2020-08-04)
 -------------------

etc/desitarget.module

@@ -77,6 +77,7 @@ setenv [string toupper $product] $PRODUCT_DIR
 # Add any non-standard Module code below this point.
 #
 setenv TARG_DIR /global/cfs/cdirs/desi/target/catalogs
+setenv MTL_DIR /global/cfs/cdirs/desi/target/mtl
 setenv DUST_DIR /global/cfs/cdirs/cosmo/data/dust/v0_1
 setenv CMX_DIR /global/cfs/cdirs/desi/target/cmx_files
 setenv SCND_DIR /global/cfs/cdirs/desi/target/secondary

py/desitarget/QA.py

@@ -662,7 +662,7 @@ def qasystematics_scatterplot(pixmap, syscolname, targcolname, qadir='.',
 
     # ADM apply the digitization to the target density values
     # ADM note that the first digitized bin is 1 not zero.
-    ### meds = [np.median(yy[wbin == bin]) for bin in range(1, nbins+1)]
+    # meds = [np.median(yy[wbin == bin]) for bin in range(1, nbins+1)]
     meds = list()
     for bin in range(1, nbins+1):
         ii = (wbin == bin)
@@ -1170,7 +1170,7 @@ def mock_qanz(cat, objtype, qadir='.', area=1.0, dndz=None, nobjscut=1000,
         templatesubtypes = np.char.strip(np.char.decode(cat['TEMPLATESUBTYPE']))
         truespectypes = np.char.strip(np.char.decode(cat['TRUESPECTYPE']))
     else:
-        #- Already a string; just strip whitespace
+        # Already a string; just strip whitespace.
         templatetypes = np.char.strip(cat['TEMPLATETYPE'])
         templatesubtypes = np.char.strip(cat['TEMPLATESUBTYPE'])
         truespectypes = np.char.strip(cat['TRUESPECTYPE'])
@@ -1485,7 +1485,7 @@ def qso_colorbox(ax, plottype='grz', verts=None):
                 templatesubtypes = np.char.strip(np.char.decode(cat['TEMPLATESUBTYPE']))
                 truespectypes = np.char.strip(np.char.decode(cat['TRUESPECTYPE']))
             else:
-                #- Already a string, just strip whitespace
+                # Already a string, just strip whitespace.
                 templatetypes = np.char.strip(cat['TEMPLATETYPE'])
                 templatesubtypes = np.char.strip(cat['TEMPLATESUBTYPE'])
                 truespectypes = np.char.strip(cat['TRUESPECTYPE'])

py/desitarget/cuts.py

@@ -37,7 +37,6 @@
 from desitarget.geomask import box_area, hp_in_box, is_in_box, is_in_hp
 from desitarget.geomask import cap_area, hp_in_cap, is_in_cap
 
-
 # ADM set up the DESI default logger
 from desiutil.log import get_logger
 log = get_logger()
@@ -2519,7 +2518,7 @@ def _select_targets_file(filename):
     def _update_status(result):
         ''' wrapper function for the critical reduction operation,
             that occurs on the main parallel process '''
-        if nbrick % 50 == 0 and nbrick > 0:
+        if nbrick % 20 == 0 and nbrick > 0:
             elapsed = time() - t0
             rate = elapsed / nbrick
             log.info('{}/{} files; {:.1f} secs/file; {:.1f} total mins elapsed'

py/desitarget/geomask.py

@@ -538,7 +538,6 @@ def sphere_circle_ra_off(theta, centdec, declocs):
           in theta from -90o->90o, which corresponds to offsets in the POSITIVE RA direction. The user must determine
           which offsets are to the negative side of the circle, or call this function twice.
     """
-
     # ADM convert the input angles from degrees to radians
     # ADM cast theta as float 64 to help deal with the cosine of very small angles
     thetar = np.radians(theta).astype('<f8')
@@ -1248,7 +1247,7 @@ def is_in_hp(objs, nside, pixlist, radec=False):
         Array of objects. Must include at columns "RA" and "DEC".
     nside : :class:`int`
         The HEALPixel nside number (NESTED scheme).
-    pixlist : :class:`list` or `~numpy.ndarray`
+    pixlist : :class:`list` or `int` or `~numpy.ndarray`
         The list of HEALPixels in which to find objects.
     radec : :class:`bool`, optional, defaults to ``False``
         If ``True`` `objs` is an [RA, Dec] list instead of a rec array.
@@ -1258,6 +1257,9 @@ def is_in_hp(objs, nside, pixlist, radec=False):
     :class:`~numpy.ndarray`
         ``True`` for objects in pixlist, ``False`` for other objects.
     """
+    # ADM if an integer is passed, convert it to an array.
+    pixlist = np.atleast_1d(pixlist)
+
     if radec:
         ra, dec = objs
     else: