DM-23237: Handle lab GROUPID usage

python/lsst/obs/lsst/translators/latiss.py

@@ -57,6 +57,9 @@
 # RASTART/DECSTART/RAEND/DECEND used wrong telescope location before this
 RASTART_IS_BAD = Time("2020-02-01T00:00", format="isot", scale="utc")
 
+# DATE-END is not to be trusted before this date
+DATE_END_IS_BAD = Time("2020-02-01T00:00", format="isot", scale="utc")
+
 # Scaling factor radians to degrees.  Keep it simple.
 RAD2DEG = 180.0 / math.pi
 
@@ -224,14 +227,34 @@ def fix_header(cls, header):
             log.debug("%s: Forcing detector serial to %s", obsid, header["LSST_NUM"])
             modified = True
 
+        if date < DATE_END_IS_BAD:
+            # DATE-END may or may not be in TAI and may or may not be
+            # before DATE-BEG.  Simpler to clear it
+            if header.get("DATE-END"):
+                header["DATE-END"] = None
+                header["MJD-END"] = None
+
+                log.debug("%s: Clearing DATE-END as being untrustworthy", obsid)
+                modified = True
+
         # Up until a certain date GROUPID was the IMGTYPE
         if date < IMGTYPE_OKAY_DATE:
             groupId = header.get("GROUPID")
             if groupId and not groupId.startswith("test"):
                 imgType = header.get("IMGTYPE")
                 if not imgType:
+                    if "_" in groupId:
+                        # Sometimes have the form dark_0001_0002
+                        # in this case we pull the IMGTYPE off the front and
+                        # do not clear groupId (although groupId may now
+                        # repeat on different days).
+                        groupId, _ = groupId.split("_", 1)
+                    elif groupId != "FOCUS" and groupId.startswith("FOCUS"):
+                        # If it is exactly FOCUS we want groupId cleared
+                        groupId = "FOCUS"
+                    else:
+                        header["GROUPID"] = None
                     header["IMGTYPE"] = groupId
-                    header["GROUPID"] = None
                     log.debug("%s: Setting IMGTYPE to '%s' from GROUPID", obsid, header["IMGTYPE"])
                     modified = True
                 else:

tests/headers/latiss-AT_O_20190915_000037.yaml

@@ -0,0 +1,93 @@
+SIMPLE: true
+EXTEND: true
+COMMENT: '---- Checksums ----'
+ORIGIN: LSST DM Header Service
+TIMESYS: TAI
+DATE: '2019-09-15T18:13:31.752'
+DATE-OBS: '2019-09-15T18:13:32.400'
+DATE-BEG: '2019-09-15T18:13:32.400'
+DATE-END: '2019-09-15T18:13:27.435'
+MJD: 58741.7593952745
+MJD-OBS: 58741.7594027778
+MJD-BEG: 58741.7594027778
+MJD-END: 58741.7593453125
+OBSID: AT_O_20190915_000037
+GROUPID: bias_0027_0100
+OBSTYPE: null
+BUNIT: adu
+TELESCOP: LSST AuxTelescope
+INSTRUME: LATISS
+OBSERVER: LSST
+OBS-LONG: -70.749417
+OBS-LAT: -30.244639
+OBS-ELEV: 2663.0
+OBSGEO-X: 1818938.94
+OBSGEO-Y: -5208470.95
+OBSGEO-Z: -3195172.08
+RA: null
+DEC: null
+RASTART: null
+DECSTART: null
+RAEND: null
+DECEND: null
+ROTPA: null
+ROTCOORD: sky
+HASTART: null
+ELSTART: null
+AZSTART: null
+AMSTART: null
+HAEND: null
+ELEND: null
+AZEND: null
+AMEND: null
+INSTPORT: null
+ATM3PORT: null
+TRACKSYS: null
+RADESYS: null
+FOCUSZ: null
+TELCODE: AT
+CONTRLLR: O
+DAYOBS: '20190915'
+SEQNUM: 37
+CCD_MANU: ITL
+CCD_TYPE: 3800C
+CCD_SERN: '20304'
+LSST_NUM: ITL-3800C-098
+SEQCKSUM: null
+SEQNAME: null
+REBNAME: null
+CONTNUM: null
+IMAGETAG: null
+TEMP_SET: null
+CCDTEMP: -90.2856826782227
+DETSIZE: '[1:4072,1:4000]'
+OVERH: 64
+OVERV: 48
+PREH: 3
+FILTER: null
+FILTPOS: null
+GRATING: null
+GRATPOS: null
+LINSPOS: null
+EXPTIME: 0.0
+SHUTTIME: 59.449
+DARKTIME: null
+FILENAME: AT_O_20190915_000037.fits
+HEADVER: 1.0.1
+XTENSION: IMAGE
+BITPIX: 32
+NAXIS: 2
+NAXIS1: 576
+NAXIS2: 2048
+PCOUNT: 0
+GCOUNT: 1
+EXTNAME: Segment10
+INHERIT: true
+DATASEC: '[4:512,1:2000]'
+DETSEC: '[509:1,1:2000]'
+DTV1: 513
+DTV2: 0
+DTM1_1: -1.0
+DTM2_2: 1.0
+DTM1_2: 0
+DTM2_1: 0

tests/test_translator.py

@@ -284,6 +284,29 @@ def test_latiss_translator(self):
                            temperature=None,
                            visit_id=1654305000120024,
                            )),
+                     ("latiss-AT_O_20190915_000037.yaml",
+                      dict(telescope="LSSTAuxTel",
+                           instrument="LATISS",
+                           boresight_rotation_coord="unknown",
+                           dark_time=0.0*u.s,
+                           detector_exposure_id=2019091500037,
+                           detector_group="RXX",
+                           detector_name="S00",
+                           detector_num=0,
+                           detector_serial="ITL-3800C-068",
+                           exposure_id=2019091500037,
+                           exposure_group="bias_0027_0100",
+                           exposure_time=0.0*u.s,
+                           object="UNKNOWN",
+                           observation_id="AT_O_20190915_000037",
+                           observation_type="bias",
+                           physical_filter="NONE",
+                           pressure=None,
+                           relative_humidity=None,
+                           science_program="unknown",
+                           temperature=None,
+                           visit_id=3575576933793566714,
+                           )),
                      ("latiss-AT_O_20191031_000004.yaml",
                       dict(telescope="LSSTAuxTel",
                            instrument="LATISS",