DM-22550: Fix some header translations for LATISS data

corrections/LATISS-AT_O_20191114_000001.yaml

@@ -0,0 +1 @@
+FILTER: diffuser

corrections/LATISS-AT_O_20191118_000001.yaml

@@ -0,0 +1 @@
+FILTER: diffuser

corrections/LATISS-AT_O_20191118_000002.yaml

@@ -0,0 +1 @@
+FILTER: diffuser

corrections/LATISS-AT_O_20191118_000003.yaml

@@ -0,0 +1 @@
+FILTER: diffuser

corrections/LATISS-AT_O_20191118_000004.yaml

@@ -0,0 +1 @@
+FILTER: diffuser

corrections/LATISS-AT_O_20191118_000006.yaml

@@ -0,0 +1 @@
+FILTER: diffuser

corrections/LATISS-AT_O_20191118_000009.yaml

@@ -0,0 +1 @@
+FILTER: diffuser

corrections/LATISS-AT_O_20191118_000011.yaml

@@ -0,0 +1 @@
+FILTER: diffuser

corrections/LATISS-AT_O_20191118_000012.yaml

@@ -0,0 +1 @@
+FILTER: diffuser

corrections/LATISS-AT_O_20191118_000013.yaml

@@ -0,0 +1 @@
+FILTER: diffuser

corrections/LATISS-AT_O_20191118_000014.yaml

@@ -0,0 +1 @@
+FILTER: diffuser

corrections/LATISS-AT_O_20191118_000016.yaml

@@ -0,0 +1 @@
+FILTER: diffuser

corrections/LATISS-AT_O_20191118_000019.yaml

@@ -0,0 +1 @@
+FILTER: diffuser

corrections/LATISS-AT_O_20191118_000021.yaml

@@ -0,0 +1 @@
+FILTER: diffuser

corrections/LATISS-AT_O_20191118_000024.yaml

@@ -0,0 +1 @@
+FILTER: diffuser

corrections/LATISS-AT_O_20191119_000001.yaml

@@ -0,0 +1 @@
+FILTER: diffuser

corrections/LATISS-AT_O_20191120_000001.yaml

@@ -0,0 +1 @@
+FILTER: diffuser

corrections/LATISS-AT_O_20191120_000002.yaml

@@ -0,0 +1 @@
+FILTER: diffuser

corrections/LATISS-AT_O_20191120_000003.yaml

@@ -0,0 +1 @@
+FILTER: diffuser

corrections/LATISS-AT_O_20191120_000004.yaml

@@ -0,0 +1 @@
+FILTER: diffuser

corrections/LATISS-AT_O_20191120_000005.yaml

@@ -0,0 +1 @@
+FILTER: diffuser

python/lsst/obs/lsst/assembly.py

@@ -91,7 +91,10 @@ def fixAmpGeometry(inAmp, bbox, metadata, logCmd=None):
         with just a change in overscan.
     """
     if logCmd is None:
-        logCmd = lambda x, *args: None  # noqa
+        # Define a null log command
+        def logCmd(*args):
+            return
+
     modified = False
 
     outAmp = inAmp.rebuild()

python/lsst/obs/lsst/filters.py

@@ -53,3 +53,33 @@
                      abstract_filter="y",
                      lambdaEff=971.68, lambdaMin=975.0, lambdaMax=1075.0, alias=['y4']),
 )
+
+LATISS_FILTER_DEFINITIONS = FilterDefinitionCollection(
+    FilterDefinition(physical_filter="NONE",
+                     lambdaEff=0.0,
+                     alias={"no_filter", "OPEN"}),
+    # The LSST Filters from L. Jones 04/07/10
+    FilterDefinition(physical_filter="u",
+                     abstract_filter="u",
+                     lambdaEff=364.59, lambdaMin=324.0, lambdaMax=395.0),
+    FilterDefinition(physical_filter="g",
+                     abstract_filter="g",
+                     lambdaEff=476.31, lambdaMin=405.0, lambdaMax=552.0),
+    FilterDefinition(physical_filter="r",
+                     abstract_filter="r",
+                     lambdaEff=619.42, lambdaMin=552.0, lambdaMax=691.0),
+    FilterDefinition(physical_filter="i",
+                     abstract_filter="i",
+                     lambdaEff=752.06, lambdaMin=818.0, lambdaMax=921.0),
+    FilterDefinition(physical_filter="z",
+                     abstract_filter="z",
+                     lambdaEff=866.85, lambdaMin=922.0, lambdaMax=997.0),
+    # official y filter
+    FilterDefinition(physical_filter="y",
+                     abstract_filter="y",
+                     lambdaEff=971.68, lambdaMin=975.0, lambdaMax=1075.0, alias=['y4']),
+    FilterDefinition(physical_filter="diffuser",
+                     lambdaEff=0.0),
+    FilterDefinition(physical_filter="EMPTY",
+                     lambdaEff=0.0),
+)

python/lsst/obs/lsst/gen3/instrument.py

@@ -30,7 +30,7 @@
 from lsst.obs.base.instrument import Instrument, addUnboundedCalibrationLabel
 from lsst.daf.butler import DatasetType
 from lsst.pipe.tasks.read_curated_calibs import read_all
-from ..filters import LSSTCAM_FILTER_DEFINITIONS
+from ..filters import LSSTCAM_FILTER_DEFINITIONS, LATISS_FILTER_DEFINITIONS
 
 from ..translators import LsstLatissTranslator, LsstCamTranslator, \
     LsstUCDCamTranslator, LsstTS3Translator, LsstComCamTranslator, \
@@ -286,7 +286,7 @@ def getRawFormatter(self, dataId):
 class LatissInstrument(LsstCamInstrument):
     """Gen3 Butler specialization for AuxTel LATISS data.
     """
-
+    filterDefinitions = LATISS_FILTER_DEFINITIONS
     instrument = "LATISS"
     policyName = "latiss"
     translatorClass = LsstLatissTranslator

python/lsst/obs/lsst/latiss.py

@@ -26,6 +26,7 @@
 from . import LsstCamMapper, LsstCamMakeRawVisitInfo
 from .ingest import LsstCamParseTask
 from .translators import LsstLatissTranslator
+from .filters import LATISS_FILTER_DEFINITIONS
 
 __all__ = ["LatissMapper", "LatissParseTask"]
 
@@ -42,6 +43,7 @@ class LatissMapper(LsstCamMapper):
 
     _cameraName = "latiss"
     yamlFileList = ["latiss/latissMapper.yaml"] + list(LsstCamMapper.yamlFileList)
+    filterDefinitions = LATISS_FILTER_DEFINITIONS
 
     def _extractDetectorName(self, dataId):
         return f"{LsstLatissTranslator.DETECTOR_GROUP_NAME}_{LsstLatissTranslator.DETECTOR_NAME}"

python/lsst/obs/lsst/lsstCamMapper.py

@@ -99,6 +99,7 @@ class LsstCamBaseMapper(CameraMapper):
     #
     MakeRawVisitInfoClass = None
     translatorClass = None
+    filterDefinitions = LSSTCAM_FILTER_DEFINITIONS
 
     def __init__(self, inputPolicy=None, **kwargs):
         #
@@ -136,7 +137,8 @@ def __init__(self, inputPolicy=None, **kwargs):
         for d in (self.mappings, self.exposures):
             d['raw'] = d['_raw']
 
-        LSSTCAM_FILTER_DEFINITIONS.defineFilters()
+        self.filterDefinitions.reset()
+        self.filterDefinitions.defineFilters()
 
         LsstCamMapper._nbit_tract = 16
         LsstCamMapper._nbit_patch = 5

python/lsst/obs/lsst/script/generateCamera.py

@@ -223,8 +223,7 @@ def indent():
                 raftYaw = perRaftData['yaw']
             except KeyError:
                 raftYaw = 0.
-            geometryWithinRaft = raftCcdData['geometryWithinRaft'] \
-                                 if 'geometryWithinRaft' in raftCcdData else {} # noqa E127
+            geometryWithinRaft = raftCcdData.get('geometryWithinRaft', {})
 
             for ccdName, ccdLayout in ccds.items():
                 if ccdName in geometryWithinRaft:

python/lsst/obs/lsst/script/rewrite_ts8_qe_files.py

@@ -62,9 +62,9 @@ def convert_qe_curve(filename):
     with a curve per amp.  If ther are other formats, a different
     converter will be necessary.
     """
-    hdu_list = fits.open(filename)
-    # qe data is in first extension
-    data = hdu_list[1].data
+    with fits.open(filename) as hdu_list:
+        # qe data is in first extension
+        data = hdu_list[1].data
     wlength = []
     eff = dict()
     for row in data:
@@ -112,8 +112,8 @@ def rewrite_ts8_files(picklefile, out_root='.', valid_start='1970-01-01T00:00:00
         full_raft_name = pickle.load(fh)
         # The pickle file was written with sequential dumps,
         # so it needs to be read sequentially as well.
-        res = pickle.load(fh)  # noqa: F841
-        detector_list = pickle.load(fh)  # noqa: F841
+        _ = pickle.load(fh)  # res
+        _ = pickle.load(fh)  # Detector list
         file_list = pickle.load(fh)
 
     for detector_name, f in file_list.items():

python/lsst/obs/lsst/translators/latiss.py

@@ -13,6 +13,7 @@
 __all__ = ("LsstLatissTranslator", )
 
 import logging
+import re
 
 import astropy.units as u
 from astropy.time import Time
@@ -40,6 +41,12 @@
 _DETECTOR_GROUP_NAME = "RXX"
 _DETECTOR_NAME = "S00"
 
+# Date 068 detector was put in LATISS
+DETECTOR_068_DATE = Time("2019-06-24T00:00", format="isot", scale="utc")
+
+# IMGTYPE header is filled in after this date
+IMGTYPE_OKAY_DATE = Time("2019-11-07T00:00", format="isot", scale="utc")
+
 
 def is_non_science_or_lab(self):
     """Pseudo method to determine whether this is a lab or non-science
@@ -145,6 +152,84 @@ def can_translate(cls, header, filename=None):
             return True
         return False
 
+    @classmethod
+    def fix_header(cls, header):
+        """Fix an incorrect LATISS header.
+
+        Parameters
+        ----------
+        header : `dict`
+            The header to update.  Updates are in place.
+
+        Returns
+        -------
+        modified = `bool`
+            Returns `True` if the header was updated.
+
+        Notes
+        -----
+        This method does not apply per-obsid corrections.  The following
+        corrections are applied:
+
+        * On June 24th 2019 the detector was changed from ITL-3800C-098
+          to ITL-3800C-068.  The header is intended to be correct in the
+          future.
+        * In late 2019 the DATE-OBS and MJD-OBS headers were reporting
+          1970 dates.  To correct, the DATE/MJD headers are copied in to
+          replace them and the -END headers are cleared.
+        * Until November 2019 the IMGTYPE was set in the GROUPID header.
+          The value is moved to IMGTYPE.
+        * SHUTTIME is always forced to be `None`.
+
+        Corrections are reported as debug level log messages.
+        """
+        modified = False
+
+        obsid = header.get("OBSID", "unknown")
+
+        # The DATE-OBS / MJD-OBS keys can be 1970
+        if header["DATE-OBS"].startswith("1970"):
+            # Copy the headers from the DATE and MJD since we have no other
+            # choice.
+            header["DATE-OBS"] = header["DATE"]
+            header["DATE-BEG"] = header["DATE-OBS"]
+            header["MJD-OBS"] = header["MJD"]
+            header["MJD-BEG"] = header["MJD-OBS"]
+
+            # And clear the DATE-END and MJD-END -- the translator will use
+            # EXPTIME instead.
+            header["DATE-END"] = None
+            header["MJD-END"] = None
+
+            log.debug("%s: Forcing 1970 dates to '%s'", obsid, header["DATE"])
+            modified = True
+
+        # Create a translator since we need the date
+        translator = cls(header)
+        date = translator.to_datetime_begin()
+        if date > DETECTOR_068_DATE:
+            header["LSST_NUM"] = "ITL-3800C-068"
+            log.debug("%s: Forcing detector serial to %s", obsid, header["LSST_NUM"])
+            modified = True
+
+        # Up until a certain date GROUPID was the IMGTYPE
+        if date < IMGTYPE_OKAY_DATE:
+            groupId = header.get("GROUPID")
+            if groupId and not groupId.startswith("test"):
+                imgType = header.get("IMGTYPE")
+                if not imgType:
+                    header["IMGTYPE"] = groupId
+                    header["GROUPID"] = None
+                    log.debug("%s: Setting IMGTYPE from GROUPID", obsid)
+                    modified = True
+
+        if header.get("SHUTTIME"):
+            log.debug("%s: Forcing SHUTTIME header to be None", obsid)
+            header["SHUTTIME"] = None
+            modified = True
+
+        return modified
+
     def _is_on_mountain(self):
         date = self.to_datetime_begin()
         if date > TSTART:
@@ -185,11 +270,23 @@ def to_location(self):
     @cache_translation
     def to_dark_time(self):
         # Docstring will be inherited. Property defined in properties.py
+
+        # Always compare with exposure time
+        # We may revisit this later if there is a cutoff date where we
+        # can always trust the header.
+        exptime = self.to_exposure_time()
+
         if self.is_key_ok("DARKTIME"):
-            return self.quantity_from_card("DARKTIME", u.s)
+            darktime = self.quantity_from_card("DARKTIME", u.s)
+            if darktime >= exptime:
+                return darktime
+            reason = "Dark time less than exposure time."
+        else:
+            reason = "Dark time not defined."
 
-        log.warning("Explicit dark time not found, setting dark time to the exposure time.")
-        return self.to_exposure_time()
+        log.warning("%s: %s Setting dark time to the exposure time.",
+                    self.to_observation_id(), reason)
+        return exptime
 
     @cache_translation
     def to_exposure_time(self):
@@ -202,7 +299,8 @@ def to_exposure_time(self):
 
         # A missing or undefined EXPTIME is problematic. Set to -1
         # to indicate that none was found.
-        log.warning("Insufficient information to derive exposure time. Setting to -1.0s")
+        log.warning("%s: Insufficient information to derive exposure time. Setting to -1.0s",
+                    self.to_observation_id())
         return -1.0 * u.s
 
     @cache_translation
@@ -219,22 +317,28 @@ def to_observation_type(self):
         """
 
         # LATISS observation type is documented to appear in OBSTYPE
-        # but for historical reasons prefers IMGTYPE.  Some data puts
-        # it in GROUPID (which is meant to be for something else).
+        # but for historical reasons prefers IMGTYPE.
         # Test the keys in order until we find one that contains a
         # defined value.
         obstype_keys = ["OBSTYPE", "IMGTYPE"]
 
-        # For now, hope that GROUPID does not contain an obs type value
-        # when on the mountain.
-        if not self._is_on_mountain():
-            obstype_keys.append("GROUPID")
-
+        obstype = None
         for k in obstype_keys:
             if self.is_key_ok(k):
                 obstype = self._header[k]
                 self._used_these_cards(k)
-                return obstype.lower()
+                obstype = obstype.lower()
+                break
+
+        if obstype is not None:
+            if obstype == "object" and not self._is_on_mountain():
+                # Do not map object to science in lab since most
+                # code assume science is on sky with RA/Dec.
+                obstype = "labobject"
+            elif obstype in ("skyexp", "object"):
+                obstype = "science"
+
+            return obstype
 
         # In the absence of any observation type information, return
         # unknown unless we think it might be a bias.
@@ -243,5 +347,26 @@ def to_observation_type(self):
             obstype = "bias"
         else:
             obstype = "unknown"
-        log.warning("Unable to determine observation type. Guessing '%s'", obstype)
+        log.warning("%s: Unable to determine observation type. Guessing '%s'",
+                    self.to_observation_id(), obstype)
         return obstype
+
+    @cache_translation
+    def to_physical_filter(self):
+        """Calculate the physical filter name.
+
+        Returns
+        -------
+        filter : `str`
+            Name of filter. Can be a combination of FILTER, FILTER1 and FILTER2
+            headers joined by a "+".  Returns "NONE" if no filter is declared.
+            Uses "EMPTY" if any of the filters indicate an "empty_N" name.
+        """
+        # The base class definition is fine
+        physical_filter = super().to_physical_filter()
+
+        # empty_N maps to EMPTY at the start of a filter concatenation
+        if physical_filter.startswith("empty"):
+            physical_filter = re.sub(r"^empty_\d+", "EMPTY", physical_filter)
+
+        return physical_filter