Merge pull request #173 from lsst/tickets/DM-23362

DM-23362: Support ancient LATISS data that lacks OBSID header

python/lsst/obs/lsst/translators/latiss.py

@@ -117,7 +117,7 @@ class LsstLatissTranslator(LsstBaseTranslator):
     }
 
     _trivial_map = {
-        "observation_id": ("OBSID", dict(default=None, checker=is_non_science)),
+        "observation_id": (["OBSID", "IMGNAME"], dict(default=None, checker=is_non_science)),
         "detector_serial": ["LSST_NUM", "DETSER"],
         "object": ("OBJECT", dict(checker=is_non_science_or_lab, default="UNKNOWN")),
         "boresight_rotation_angle": (["ROTPA", "ROTANGLE"], dict(checker=is_non_science_or_lab,
@@ -199,7 +199,42 @@ def fix_header(cls, header):
         """
         modified = False
 
-        obsid = header.get("OBSID", "unknown")
+        if "OBSID" not in header:
+            # Very old data used IMGNAME
+            header["OBSID"] = header.get("IMGNAME", "unknown")
+            modified = True
+            log.debug("Assigning OBSID to a value of '%s'", header["OBSID"])
+
+        obsid = header["OBSID"]
+
+        if "DAYOBS" not in header:
+            # OBS-NITE could have the value for DAYOBS but it is safer
+            # for older data to set it from the OBSID. Fall back to OBS-NITE
+            # if we have no alternative
+            dayObs = None
+            try:
+                dayObs = obsid.split("_", 3)[2]
+            except ValueError:
+                # did not split as expected
+                pass
+            if dayObs is None or len(dayObs) != 8:
+                dayObs = header["OBS-NITE"]
+                log.debug("%s: Setting DAYOBS to '%s' from OBS-NITE header", obsid, dayObs)
+            else:
+                log.debug("%s: Setting DAYOBS to '%s' from OBSID", obsid, dayObs)
+            header["DAYOBS"] = dayObs
+            modified = True
+
+        if "SEQNUM" not in header:
+            try:
+                seqnum = obsid.split("_", 3)[3]
+            except ValueError:
+                # did not split as expected
+                pass
+            else:
+                header["SEQNUM"] = int(seqnum)
+                modified = True
+                log.debug("%s: Extracting SEQNUM of '%s' from OBSID", obsid, header["SEQNUM"])
 
         # The DATE-OBS / MJD-OBS keys can be 1970
         if header["DATE-OBS"].startswith("1970"):
@@ -248,7 +283,7 @@ def fix_header(cls, header):
                         # do not clear groupId (although groupId may now
                         # repeat on different days).
                         groupId, _ = groupId.split("_", 1)
-                    elif groupId != "FOCUS" and groupId.startswith("FOCUS"):
+                    elif groupId.upper() != "FOCUS" and groupId.upper().startswith("FOCUS"):
                         # If it is exactly FOCUS we want groupId cleared
                         groupId = "FOCUS"
                     else:

tests/headers/latiss-AT_O_20190306_000014.yaml

@@ -0,0 +1,78 @@
+SIMPLE: true
+EXTEND: true
+COMMENT: '  and Astrophysics'', volume 376, page 359; bibcode: 2001A&A...376..359H'
+ORIGIN: LSST DM Header Service
+DATE: '2019-03-06T19:37:57.244'
+DATE-OBS: '2019-03-06T19:37:54.309'
+MJD: 58548.8180236578
+MJD-OBS: 58548.8179896848
+OBS-LONG: -70.749417
+OBS-LAT: -30.244639
+OBS-ELEV: 2663.0
+OBS-NITE: '20190306'
+TELESCOP: LSST AuxTelescope
+INSTRUME: LATISS
+RATEL: null
+DECTEL: null
+RA: null
+DEC: null
+ROTANGLE: null
+HA: null
+ZD: null
+AZ: null
+AIRMASS: null
+PROGRAM: null
+OBSERVER: null
+PROPOSER: null
+PROPID: null
+SEQFILE: null
+CCD_MANU: ITL
+CCD_TYPE: 3800C
+CCD_SERN: '20429'
+LSST_NUM: ITL-3800C-098
+RAFTNAME: LCA-11021_RTM-004_ETU2-Dev
+SEQNAME: null
+IMGNAME: AT_O_20190306_000014
+IMGTYPE: null
+OVERH: 64
+OVERV: 48
+PREH: 3
+TEMP_SET: null
+CCDTEMP: null
+FILTER: null
+FILTPOS1: null
+GRATING: null
+FILTPOS2: null
+BUNIT: adu
+EXPTIME: 1.06
+SHUT_DEL: null
+FILENAME: AT_O_20190306_000014.fits
+HEADVER: ACCS_MCM/OCS_Bridge
+CCDGAIN: null
+CCDNOISE: null
+MONDIODE: null
+CCDBSS: null
+XTENSION: IMAGE
+BITPIX: 32
+NAXIS: 2
+NAXIS1: 576
+NAXIS2: 2048
+PCOUNT: 0
+GCOUNT: 1
+EXTNAME: Segment10
+INHERIT: true
+CHANNEL: 9
+CCDSUM: 1 1
+AVERAGE: null
+AVGBIAS: null
+STDVBIAS: null
+STDEV: null
+DATASEC: '[4:512,1:2000]'
+DETSEC: '[509:1,1:2000]'
+DETSIZE: '[1:4072,1:4000]'
+DTV1: 513
+DTV2: 0
+DTM1_1: -1.0
+DTM2_2: 1.0
+DTM1_2: 0
+DTM2_1: 0

tests/test_translator.py

@@ -238,6 +238,29 @@ def test_latiss_translator(self):
                            temperature=None,
                            visit_id=3018092000065,
                            )),
+                     ("latiss-AT_O_20190306_000014.yaml",
+                      dict(telescope="LSSTAuxTel",
+                           instrument="LATISS",
+                           boresight_rotation_coord="unknown",
+                           dark_time=1.06*u.s,
+                           detector_exposure_id=2019030600014,
+                           detector_group="RXX",
+                           detector_name="S00",
+                           detector_num=0,
+                           detector_serial="ITL-3800C-098",
+                           exposure_id=2019030600014,
+                           exposure_group="2019030600014",
+                           exposure_time=1.06*u.s,
+                           object="UNKNOWN",
+                           observation_id="AT_O_20190306_000014",
+                           observation_type="unknown",
+                           physical_filter="????~????",
+                           pressure=None,
+                           relative_humidity=None,
+                           science_program="unknown",
+                           temperature=None,
+                           visit_id=2019030600014,
+                           )),
                      ("latiss-AT_O_20190329_000022-ats-wfs_ccd.yaml",
                       dict(telescope="LSSTAuxTel",
                            instrument="LATISS",