Merge pull request #176 from desihub/bitmasks

Expanded bitmasks docs

bin/update_bitmasks

@@ -0,0 +1,336 @@
+#!/usr/bin/env python
+
+"""
+Update doc/bitmasks.rst based upon code descriptions
+"""
+
+bitmask_template = """
+=================
+Bit Masks in DESI
+=================
+
+.. NOTE: this file is generated by desidatamodel/bin/update_bitmasks, e.g.
+..   cd desidatamodel/doc
+..   python ../bin/update_bitmasks > bitmasks.rst && make html
+
+This page describes the bitmasks found in DESI files.
+For details on working with these values, please see the tutorial_ on
+that topic.
+
+Redshift Fitting (Redrock) Masks
+--------------------------------
+
+ZWARN
+~~~~~
+
+The **ZWARN** bitmask in redshift catalogs indicates known problems with a
+particular redshift fit or associated QA.
+ZWARN==0 is good; any non-zero value indicates a potential problem.
+This mask will be described in more detail in the Redrock paper
+(Bailey et al 2023 in prep), as well as Section 5.3.1 of the
+Survey Operations Paper (Schlafly et al 2023 TODO: add link).
+
+The canonical code location defining these bits is
+`desitarget targetmask.yaml <https://github.com/desihub/desitarget/blob/main/py/desitarget/data/targetmask.yaml#L230>`_.
+Bits 0-15 are set by Redrock itself (the redshift fitter),
+while bits 16-19 are set by DESI-specific post-processing.
+
+ZWARN Mask Locations
+^^^^^^^^^^^^^^^^^^^^
+
+================ ============= ===========
+File             Table HDU     Column
+================ ============= ===========
+|redrock|        REDSHIFTS     ZWARN
+|emline|         EMLINEFIT     ZWARN
+|zmtl|           ZMTL          ZWARN
+|mtl|            MTL           ZWARN
+|zpix|           ZCATALOG      ZWARN
+|ztile|          ZCATALOG      ZWARN
+|zall|           ZCATALOG      ZWARN
+|lss|            LSS           ZWARN
+================ ============= ===========
+
+.. |redrock| replace:: :doc:`redrock <DESI_SPECTRO_REDUX/SPECPROD/tiles/GROUPTYPE/TILEID/GROUPID/redrock-SPECTROGRAPH-TILEID-GROUPID>`
+.. |emline| replace:: :doc:`emline <DESI_SPECTRO_REDUX/SPECPROD/tiles/GROUPTYPE/TILEID/GROUPID/emline-SPECTROGRAPH-TILEID-GROUPID>`
+.. |zmtl| replace:: :doc:`zmtl <DESI_SPECTRO_REDUX/SPECPROD/tiles/GROUPTYPE/TILEID/GROUPID/zmtl-SPECTROGRAPH-TILEID-GROUPID>`
+.. |mtl| replace:: :doc:`mtl <DESI_SURVEYOPS/mtl/main/dark/mtl-dark-hp-HPX>`
+.. |zpix| replace:: :doc:`zpix <DESI_SPECTRO_REDUX/SPECPROD/zcatalog/zpix-SURVEY-PROGRAM>`
+.. |ztile| replace:: :doc:`ztile <DESI_SPECTRO_REDUX/SPECPROD/zcatalog/ztile-SURVEY-PROGRAM-GROUPTYPE>`
+.. |zall| replace:: :doc:`zall <DESI_SPECTRO_REDUX/SPECPROD/zcatalog/zall-pix-SPECPROD>`
+.. |lss| replace:: :doc:`lss <DESI_ROOT/vac/RELEASE/lss/VERSION/LSScats/full/fullVETO_dat>`
+
+ZWARN Bit Definitions
+^^^^^^^^^^^^^^^^^^^^^
+
+{ZWARN}
+
+Spectroscopic Reduction Masks
+-----------------------------
+
+The **FIBERSTATUS** bit mask records the state of individual
+fibers for issues that impact the entire spectrum, e.g. a broken fiber.
+The **SPECMASK** bit mask tracks wavelength dependent isses per spectrum,
+e.g. masks for cosmic rays.
+
+FIBERSTATUS
+~~~~~~~~~~~
+
+The **FIBERSTATUS** mask is kept as a column in FIBERMAP and related HDUs.
+Bits 0-7 are set by fiber assignment from focal plane information known
+before observations; bits 8-24 are set by the spectroscopic pipeline;
+bits 25-30 are set by the final QA step to set bits for all fibers in
+a petal (e.g. because sky model noise makes all spectra questionable).
+
+The canonical code location defining FIBERSTATUS bits is
+`desispec.maskbits L55 <https://github.com/desihub/desispec/blob/0.55.0/py/desispec/maskbits.py#L55>`_.
+
+FIBERSTATUS Mask Locations
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+================ ============= ===========
+File             Table HDU     Column
+================ ============= ===========
+|frame|          FIBERMAP      FIBERSTATUS
+|sframe|         FIBERMAP      FIBERSTATUS
+|cframe|         FIBERMAP      FIBERSTATUS
+|spectra|        FIBERMAP      FIBERSTATUS
+|coadd|          EXP_FIBERMAP  FIBERSTATUS
+|coadd|          FIBERMAP      COADD_FIBERSTATUS
+|exposure-qa|    FIBERQA       QAFIBERSTATUS
+|tile-qa|        FIBERQA       QAFIBERSTATUS
+================ ============= ===========
+
+.. |frame| replace:: :doc:`frame <DESI_SPECTRO_REDUX/SPECPROD/exposures/NIGHT/EXPID/frame-CAMERA-EXPID>`
+.. |sframe| replace:: :doc:`sframe <DESI_SPECTRO_REDUX/SPECPROD/exposures/NIGHT/EXPID/sframe-CAMERA-EXPID>`
+.. |cframe| replace:: :doc:`cframe <DESI_SPECTRO_REDUX/SPECPROD/exposures/NIGHT/EXPID/cframe-CAMERA-EXPID>`
+.. |spectra| replace:: :doc:`spectra <DESI_SPECTRO_REDUX/SPECPROD/healpix/SURVEY/PROGRAM/PIXGROUP/PIXNUM/spectra-SURVEY-PROGRAM-PIXNUM>`
+.. |coadd| replace:: :doc:`coadd <DESI_SPECTRO_REDUX/SPECPROD/healpix/SURVEY/PROGRAM/PIXGROUP/PIXNUM/coadd-SURVEY-PROGRAM-PIXNUM>`
+.. |exposure-qa| replace:: :doc:`exposure-qa <DESI_SPECTRO_REDUX/SPECPROD/exposures/NIGHT/EXPID/exposure-qa-EXPID>`
+.. |tile-qa| replace:: :doc:`tile-qa <DESI_SPECTRO_REDUX/SPECPROD/tiles/GROUPTYPE/TILEID/GROUPID/tile-qa-TILEID-GROUPID>`
+
+FIBERSTATUS Bit Definitions
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Bit 3 (RESTRICTED) is informative and doesn't necessarily mean that the spectrum is bad,
+i.e. a FIBERSTATUS value of 0 or 8=2**3 is good.
+
+{FIBERSTATUS}
+
+SPECMASK
+~~~~~~~~
+
+The **SPECMASK** is stored as an image HDU in files with spectra,
+matched to the FLUX HDU, i.e. specmask[i,j] is the mask for fiber i
+wavelength j with flux value flux[i,j].  All bits in SPECMASK are bad,
+i.e. non-zero values mean that the corresponding flux should not be used.
+
+The canonical code location defining SPECMASK bits is
+`desispec.maskbits L84 <https://github.com/desihub/desispec/blob/0.55.0/py/desispec/maskbits.py#L84>`_.
+
+SPECMASK Mask Locations
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Note: the FITS file HDU EXTNAME=MASK or MASK_B/R/Z, not "SPECMASK".
+
+================ =============
+File             Image HDU
+================ =============
+|frame|          MASK
+|sframe|         MASK
+|cframe|         MASK
+|spectra|        MASK_B/R/Z
+|coadd|          MASK_B/R/Z
+================ =============
+
+SPECMASK Bit Definitions
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+{SPECMASK}
+
+CCDMASK
+~~~~~~~
+
+The **CCDMASK** is used for masking spectrograph CCD images during preprocessing,
+prior to extracting the spectra.  It is stored in the MASK HDU of
+:doc:`preproc <DESI_SPECTRO_REDUX/SPECPROD/preproc/NIGHT/EXPID/preproc-CAMERA-EXPID>` files.
+
+The canonical code location defining CCDMASK bits is
+`desispec.maskbits L42 <https://github.com/desihub/desispec/blob/0.55.0/py/desispec/maskbits.py#L42>`_.
+
+CCDMASK Bit Definitions
+^^^^^^^^^^^^^^^^^^^^^^^
+
+{CCDMASK}
+
+Target masks
+------------
+
+Target masks record the reasons why each target was selected for DESI
+observations.  These are stored in the ``*_TARGET`` columns of the
+TARGETS, FIBERASSIGN, and FIBERMAP tables in data files.
+
+These masks are described in more detail in Section 2 of
+`Myers et al. (2023) <https://ui.adsabs.harvard.edu/abs/2023AJ....165...50M/abstract>`_
+and Appendices A and B of the the DESI EDR Overview paper
+(DESI Collaboration et al 2023 TODO: add link).
+
+The following table lists a subset of the most commonly used bits that maintained
+the same definition throughout different phases of DESI observations.  For the
+full definition of all bits, see the EDR Overview paper appendices and the
+code links in the second table below.
+
+==================== ========== ===========
+Bit Name             Bit Number Description
+==================== ========== ===========
+LRG                  0          Luminous Red Galaxies
+ELG                  1          Emission Line Galaxies
+QSO                  2          Quasars
+SKY                  32         Blank sky locations
+(various STD_*)      33-35      Standard stars
+BGS_ANY              60         Bright Galaxy Survey galaxies
+MWS_ANY              61         Milky Way Survey stars
+SCND_ANY             62         Secondary targets
+==================== ========== ===========
+
+
+Canonical code locations where targeting bits are defined:
+
+=================================== ==================
+BIT_MASK                            URL
+=================================== ==================
+CMX_TARGET                          `CMX`_
+SV1_DESI_TARGET                     `SV1`_
+SV1_BGS_TARGET                      `SV1`_
+SV1_MWS_TARGET                      `SV1`_
+SV2_DESI_TARGET                     `SV2`_
+SV2_BGS_TARGET                      `SV2`_
+SV2_MWS_TARGET                      `SV2`_
+SV2_SCND_TARGET                     `SV2`_
+SV3_DESI_TARGET                     `SV3`_
+SV3_BGS_TARGET                      `SV3`_
+SV3_MWS_TARGET                      `SV3`_
+SV3_SCND_TARGET                     `SV3`_
+DESI_TARGET                         `TARGET`_
+BGS_TARGET                          `TARGET`_
+MWS_TARGET                          `TARGET`_
+SCND_TARGET                         `TARGET`_
+OBSCONDITIONS                       `TARGET_L188`_
+=================================== ==================
+
+
+Imaging masks
+-------------
+
+These masks were defined or used by the `DESI Legacy Imaging Surveys`_. Please
+see their information on these masks at the links below.
+
+=================================== ==================
+BIT_MASK                            URL
+=================================== ==================
+WISEMASK_W1                         `BITMASKS_LEGACY`_
+WISEMASK_W2                         `BITMASKS_LEGACY`_
+MASKBITS                            `BITMASKS_LEGACY`_
+=================================== ==================
+
+
+.. _`CMX`: https://github.com/desihub/desitarget/blob/2.5.0/py/desitarget/cmx/data/cmx_targetmask.yaml
+.. _`SV1`: https://github.com/desihub/desitarget/blob/2.5.0/py/desitarget/sv1/data/sv1_targetmask.yaml
+.. _`SV2`: https://github.com/desihub/desitarget/blob/2.5.0/py/desitarget/sv2/data/sv2_targetmask.yaml
+.. _`SV3`: https://github.com/desihub/desitarget/blob/2.5.0/py/desitarget/sv3/data/sv3_targetmask.yaml
+.. _`TARGET`: https://github.com/desihub/desitarget/blob/2.5.0/py/desitarget/data/targetmask.yaml
+.. _`MASKBITS_L55`: https://github.com/desihub/desispec/blob/0.55.0/py/desispec/maskbits.py#L55
+.. _`TARGET_L188`: https://github.com/desihub/desitarget/blob/2.5.0/py/desitarget/data/targetmask.yaml#L188
+.. _`MASKBITS_L84`: https://github.com/desihub/desispec/blob/0.55.0/py/desispec/maskbits.py#L84
+.. _`ZWARN`: https://github.com/desihub/redrock/blob/0.16.0/py/redrock/zwarning.py#L14
+.. _`BITMASKS_LEGACY`: https://www.legacysurvey.org/dr9/bitmasks/
+.. _`DESI Legacy Imaging Surveys`: https://www.legacysurvey.org/
+.. _tutorial: https://github.com/desihub/desitarget/blob/master/doc/nb/target-selection-bits-and-bitmasks.ipynb
+"""
+
+#-------------------------------------------------------------------------
+#- ZWARN mask from desitarget is a superset of the bits defined in Redrock
+from desitarget.targetmask import zwarn_mask
+
+ZWARN_rows = list()
+for name in zwarn_mask.names():
+    bitnum = zwarn_mask.bitnum(name)
+    comment = zwarn_mask.comment(name)
+    ZWARN_rows.append(f'{name:20s} {bitnum:10d} {comment}')
+
+ZWARN_rows = '\n'.join(ZWARN_rows)
+ZWARN = f"""
+==================== ========== ===========
+Bit Name             Bit Number Description
+==================== ========== ===========
+{ZWARN_rows}
+==================== ========== ===========
+"""
+
+#-------------------------------------------------------------------------
+#- FIBERSTATUS
+from desispec.maskbits import fibermask
+
+FIBERSTATUS_rows = list()
+for name in fibermask.names():
+    bitnum = fibermask.bitnum(name)
+    comment = fibermask.comment(name)
+    FIBERSTATUS_rows.append(f'{name:20s} {bitnum:10d} {comment}')
+
+FIBERSTATUS_rows = '\n'.join(FIBERSTATUS_rows)
+FIBERSTATUS = f"""
+==================== ========== ===========
+Bit Name             Bit Number Description
+==================== ========== ===========
+{FIBERSTATUS_rows}
+==================== ========== ===========
+"""
+
+#-------------------------------------------------------------------------
+#- SPECMASK
+from desispec.maskbits import specmask
+
+SPECMASK_rows = list()
+for name in specmask.names():
+    bitnum = specmask.bitnum(name)
+    comment = specmask.comment(name)
+    SPECMASK_rows.append(f'{name:20s} {bitnum:10d} {comment}')
+
+SPECMASK_rows = '\n'.join(SPECMASK_rows)
+SPECMASK = f"""
+==================== ========== ===========
+Bit Name             Bit Number Description
+==================== ========== ===========
+{SPECMASK_rows}
+==================== ========== ===========
+"""
+
+#-------------------------------------------------------------------------
+#- CCDMASK
+from desispec.maskbits import ccdmask
+
+CCDMASK_rows = list()
+for name in ccdmask.names():
+    bitnum = ccdmask.bitnum(name)
+    comment = ccdmask.comment(name)
+    CCDMASK_rows.append(f'{name:20s} {bitnum:10d} {comment}')
+
+CCDMASK_rows = '\n'.join(CCDMASK_rows)
+CCDMASK = f"""
+==================== ========== ===========
+Bit Name             Bit Number Description
+==================== ========== ===========
+{CCDMASK_rows}
+==================== ========== ===========
+"""
+
+
+#-------------------------------------------------------------------------
+#- Print rst
+
+print(bitmask_template.format(
+        ZWARN=ZWARN,
+        FIBERSTATUS=FIBERSTATUS,
+        SPECMASK=SPECMASK,
+        CCDMASK=CCDMASK,
+        ))