Merge bdba75f into f394557

doc/changes.rst

@@ -5,16 +5,27 @@ desitarget Change Log
 1.2.0 (unreleased)
 ------------------
 
+* Update the data model for Main Survey MTL [`PR #755`_]. Includes:
+    * Read all types of MTL ledgers, including SV3 and Main Survey 1.0.0.
+        * Addresses `issue #749`_.
+        * Also part of `PR #753`_ but I needed it to make progress.
+    * Change the location of the "ZTILE" (zdone=True/False) file:
+        * For sv it was $ZCAT_DIR/tiles.csv
+        * For main it will be $MTL_DIR/../ops/tiles-specstatus.ecsv
+    * A workaround to process SV3 files that have Main-like fibermaps.
 * Add new ZWARN bits BAD_SPECQA and BAD_PETALQA [`PR #754`_].
 * ``make_zqso_files`` more robust to individual tile,night,petal problems
   while still processing the others [`PR #752`_].
 * Add final QSO decision logic to full Main Survey MTL loop [`PR #751`_].
 * Make creation of zqso catalogs robust to missing columns [`PR #750`_].
 
+.. _`issue #749`: https://github.com/desihub/desitarget/issues/749
 .. _`PR #750`: https://github.com/desihub/desitarget/pull/750
 .. _`PR #751`: https://github.com/desihub/desitarget/pull/751
 .. _`PR #752`: https://github.com/desihub/desitarget/pull/752
+.. _`PR #753`: https://github.com/desihub/desitarget/pull/753
 .. _`PR #754`: https://github.com/desihub/desitarget/pull/754
+.. _`PR #755`: https://github.com/desihub/desitarget/pull/755
 
 1.1.1 (2021-05-29)
 ------------------

etc/desitarget.module

@@ -77,7 +77,7 @@ setenv [string toupper $product] $PRODUCT_DIR
 # Add any non-standard Module code below this point.
 #
 setenv TARG_DIR $env(DESI_ROOT)/target/catalogs
-setenv MTL_DIR  $env(DESI_ROOT)/target/mtl
+setenv MTL_DIR  $env(DESI_ROOT)/target/surveyops/mtl
 setenv DUST_DIR $env(DESI_ROOT)/external/dust/v0_1
 setenv CMX_DIR  $env(DESI_ROOT)/target/cmx_files
 setenv SCND_DIR $env(DESI_ROOT)/target/secondary

py/desitarget/io.py

@@ -2604,6 +2604,10 @@ def read_mtl_ledger(filename, unique=True, isodate=None,
         from desitarget.mtl import mtldatamodel, survey_data_model
         # ADM allow for the full set of possible columns.
         mtldm = survey_data_model(mtldatamodel, survey="main")
+        # ADM need to include two columns that were briefly included as
+        # ADM part of the ledgers in version 1.0.0 of desitarget.
+        fulldescr = mtldm.dtype.descr + [('ZS', 'U2'), ('ZINFO', 'U8')]
+        mtldm = np.array([], dtype=fulldescr)
         # ADM the data model can differ depending on survey type.
         names, forms = [], []
         with open(filename) as f:
@@ -2613,12 +2617,6 @@ def read_mtl_ledger(filename, unique=True, isodate=None,
                     iname, iform = [i+1 for i, stringy in enumerate(l) if
                                     "name" in stringy or "datatype" in stringy]
                     name, form = l[iname][:-1], l[iform][:-1]
-
-                    # SB for backwards compatibility with mtl 1.0.0 format,
-                    # SB which doesn't have Z_QN and has placeholder ZS,ZINFO
-                    if name not in mtldm.dtype.names:
-                        continue
-
                     names.append(name)
                     if 'string' in form:
                         forms.append(mtldm[name].dtype.str)

py/desitarget/lyazcat.py

@@ -646,7 +646,7 @@ def zcat_writer(zcat, outputdir, outputname,
     # ADM create the header and add the standard DESI dependencies.
     hdr = {}
     add_dependencies(hdr)
-    add_dependencies(hdr, module_names=['quasarnp',])
+    add_dependencies(hdr, module_names=['quasarnp', ])
 
     # ADM add the specific lyazcat dependencies
     hdr['QN_ADDED'] = qn_flag

py/desitarget/mtl.py

@@ -252,15 +252,27 @@ def get_mtl_tile_file_name(secondary=False):
     return fn
 
 
-def get_ztile_file_name():
+def get_ztile_file_name(survey='main'):
     """Convenience function to grab the name of the ZTILE file.
 
+    survey : :class:`str`, optional, defaults to "main"
+        To look up the correct ZTILE filename. Options are ``'main'`` and
+        ``'svX``' (where X is 1, 2, 3 etc.) for the main survey and
+        different iterations of SV, respectively.
+
     Returns
     -------
     :class:`str`
         The name of the ZTILE file.
     """
-    fn = "tiles.csv"
+    if survey[:2] == 'sv':
+        fn = "tiles.csv"
+    elif survey == 'main':
+        fn = "tiles-specstatus.ecsv"
+    else:
+        msg = "Allowed 'survey' inputs are sv(X) or main, not {}".format(survey)
+        log.critical(msg)
+        raise ValueError(msg)
 
     return fn
 
@@ -1011,7 +1023,7 @@ def tiles_to_be_processed(zcatdir, mtltilefn, obscon, survey):
     Parameters
     ----------
     zcatdir : :class:`str`
-        Full path to the "daily" directory that hosts redshift catalogs.
+        Full path to the directory that hosts redshift catalogs.
     mtltilefn : :class:`str`
         Full path to the file of tiles that have been processed by MTL.
     obscon : :class:`str`
@@ -1028,9 +1040,27 @@ def tiles_to_be_processed(zcatdir, mtltilefn, obscon, survey):
     :class:`~numpy.array`
         An array of tiles that have not yet been processed and written to
         the mtl tile file.
+
+    Notes
+    -----
+    - If `survey` is `'main'` the code assumes the file with zdone for
+      spectro tiles is `mtltilefn`/../../ops/tiles-specstatus.ecsv
+      (i.e. it is in the ops directory parallel to the mtl directory).
+      If `survey` is `'svX'` the code assumes it is `zcatdir`/tiles.csv.
     """
     # ADM read in the ZTILE file.
-    ztilefn = os.path.join(zcatdir, get_ztile_file_name())
+    ztilefn = get_ztile_file_name(survey=survey)
+    # ADM directory structure is different for SV and the Main Survey.
+    if survey[:2] == 'sv':
+        ztilefn = os.path.join(zcatdir, ztilefn)
+    elif survey == 'main':
+        opsdir = os.path.dirname(mtltilefn).replace("mtl", "ops")
+        ztilefn = os.path.join(opsdir, ztilefn)
+    else:
+        msg = "Allowed 'survey' inputs are sv(X) or main, not {}".format(survey)
+        log.critical(msg)
+        raise ValueError(msg)
+
     tilelookup = Table.read(ztilefn)
 
     # ADM the ZDONE column is a string, convert to a Boolean.
@@ -1211,6 +1241,11 @@ def make_zcat_rr_backstop(zcatdir, tiles, obscon, survey):
             allzs.append(zz)
             # ADM read in all of the exposures in the fibermap.
             fm = fitsio.read(zbestfn, "FIBERMAP")
+            # ADM in the transition between SV3 and the Main Survey, the
+            # ADM fibermap data model changed. New columns may need to be
+            # ADM removed to concatenate old- and new-style fibermaps.
+            if "PLATE_RA" in fm.dtype.names:
+                fm = rfn.drop_fields(fm, ["PLATE_RA", "PLATE_DEC"])
             # ADM recover the information for unique targets based on the
             # ADM first entry for each TARGETID.
             _, ii = np.unique(fm['TARGETID'], return_index=True)
@@ -1284,9 +1319,9 @@ def loop_ledger(obscon, survey='main', zcatdir=None, mtldir=None,
         Options are ``'main'`` and ``'svX``' (where X is 1, 2, 3 etc.)
         for the main survey and different iterations of SV, respectively.
     zcatdir : :class:`str`, optional, defaults to ``None``
-        Full path to the "daily" directory that hosts redshift catalogs.
-        If this is ``None``, look up the redshift catalog directory from
-        the $ZCAT_DIR environment variable.
+        Full path to the directory that hosts redshift catalogs. If this
+        is ``None``, look up the redshift catalog directory from the
+        $ZCAT_DIR environment variable.
     mtldir : :class:`str`, optional, defaults to ``None``
         Full path to the directory that hosts the MTL ledgers and the MTL
         tile file. If ``None``, then look up the MTL directory from the
@@ -1306,15 +1341,19 @@ def loop_ledger(obscon, survey='main', zcatdir=None, mtldir=None,
     :class:`str`
         The name of the MTL tile file that was updated.
     :class:`str`
-        The name of the ZTILE file that was used to link TILEIDs to
-        observing conditions and to determine if tiles were "done".
+        Name of ZTILE file used to link TILEIDs to observing conditions
+        to determine if tiles were "done" (that they had zdone=True).
     :class:`~numpy.array`
         Information for the tiles that were processed.
 
     Notes
     -----
     - Assumes all of the relevant ledgers have already been made by,
       e.g., :func:`~desitarget.mtl.make_ledger()`.
+    - If `survey` is `'main'` the code assumes the file with the zdone
+      status for spectro tiles is `mtldir`/../ops/tiles-specstatus.ecsv
+      (i.e. it is in the ops directory parallel to the mtl directory).
+      If `survey` is `'svX'` the code assumes it is `zcatdir`/tiles.csv.
     """
     # ADM first grab all of the relevant files.
     # ADM grab the MTL directory (in case we're relying on $MTL_DIR).
@@ -1336,12 +1375,19 @@ def loop_ledger(obscon, survey='main', zcatdir=None, mtldir=None,
                                      survey=survey, obscon=obscon, ender=form)
     # ADM grab the zcat directory (in case we're relying on $ZCAT_DIR).
     zcatdir = get_zcat_dir(zcatdir)
-    # ADM And contruct the associated ZTILE filename.
-    ztilefn = os.path.join(zcatdir, get_ztile_file_name())
 
     # ADM grab an array of tiles that are yet to be processed.
     tiles = tiles_to_be_processed(zcatdir, mtltilefn, obscon, survey)
 
+    # ADM contruct the ZTILE filename, for logging purposes.
+    ztilefn = get_ztile_file_name(survey=survey)
+    # ADM directory structure is different for SV and the Main Survey.
+    if survey[:2] == 'sv':
+        ztilefn = os.path.join(zcatdir, ztilefn)
+    elif survey == 'main':
+        opsdir = os.path.dirname(mtltilefn).replace("mtl", "ops")
+        ztilefn = os.path.join(opsdir, ztilefn)
+
     # ADM stop if there are no tiles to process.
     if len(tiles) == 0:
         return hpdirname, mtltilefn, ztilefn, tiles