DM-39347: Update TS8 translator

python/lsst/obs/lsst/translators/lsst.py

@@ -376,7 +376,7 @@ def compute_exposure_id(dayobs, seqnum, controller=None):
 
         Parameters
         ----------
-        dayobs : `str`
+        dayobs : `str` or `int`
             Day of observation in either YYYYMMDD or YYYY-MM-DD format.
             If the string looks like ISO format it will be truncated before the
             ``T`` before being handled.
@@ -396,6 +396,10 @@ def compute_exposure_id(dayobs, seqnum, controller=None):
         exposure_id : `int`
             Exposure ID in form YYYYMMDDnnnnn form.
         """
+        # We really want an integer but the checks require a str.
+        if isinstance(dayobs, int):
+            dayobs = str(dayobs)
+
         if "T" in dayobs:
             dayobs = dayobs[:dayobs.find("T")]
 

python/lsst/obs/lsst/translators/ts8.py

@@ -101,40 +101,58 @@ def can_translate(cls, header, filename=None):
 
         return False
 
-    @staticmethod
-    def compute_exposure_id(dateobs, seqnum=0, controller=None):
-        """Helper method to calculate the TS8 exposure_id.
-
-        Parameters
-        ----------
-        dateobs : `str`
-            Date of observation in FITS ISO format.
-        seqnum : `int`, unused
-            Sequence number. Ignored.
-        controller : `str`, unused
-            Controller type. Ignored.
+    @classmethod
+    def fix_header(cls, header, instrument, obsid, filename=None):
+        """Fix TS8 headers.
 
-        Returns
-        -------
-        exposure_id : `int`
-            Exposure ID.
+        Notes
+        -----
+        See `~astro_metadata_translator.fix_header` for details of the general
+        process.
         """
-        # There is worry that seconds are too coarse so use 10th of second
-        # and read the first 21 characters.
-        exposure_id = re.sub(r"\D", "", dateobs[:21])
-        return int(exposure_id)
+        modified = False
+
+        # Calculate the standard label to use for log messages
+        log_label = cls._construct_log_prefix(obsid, filename)
+
+        if header.get("DATE-OBS", "OBS") == header.get("DATE-TRG", "TRG"):
+            log.warning("%s: DATE-OBS detected referring to end of observation.", log_label)
+            if "DATE-END" not in header:
+                header["DATE-END"] = header["DATE-OBS"]
+                header["MJD-END"] = header["MJD-OBS"]
+
+            # Time system used to be UTC and at some point became TAI.
+            # Need to include the transition date and update the TIMESYS
+            # header.
+            timesys = header.get("TIMESYS", "utc").lower()
+
+            # Need to subtract exposure time from DATE-OBS.
+            date_obs = None
+            for (key, format) in (("MJD-OBS", "mjd"), ("DATE-OBS", "isot")):
+                if date_val := header.get(key):
+                    date_obs = Time(date_val, format=format, scale=timesys)
+                    break
+
+            if date_obs:
+                exptime = TimeDelta(header["EXPTIME"]*u.s, scale="tai")
+                date_obs = date_obs - exptime
+                header["MJD-OBS"] = date_obs.mjd
+                header["DATE-OBS"] = date_obs.isot
+                header["DATE-BEG"] = header["DATE-OBS"]
+                header["MJD-BEG"] = header["MJD-OBS"]
+
+                modified = True
+            else:
+                # This should never happen because DATE-OBS is already present.
+                log.warning("%s: Unexpectedly failed to extract date from DATE-OBS/MJD-OBS", log_label)
+
+        return modified
 
     @classmethod
     def compute_detector_exposure_id(cls, exposure_id, detector_num):
         # Docstring inherited from LsstBaseTranslator.
         return compute_detector_exposure_id_generic(exposure_id, detector_num, max_num=cls.DETECTOR_MAX)
 
-    @cache_translation
-    def to_datetime_begin(self):
-        # Docstring will be inherited. Property defined in properties.py
-        self._used_these_cards("MJD-OBS")
-        return Time(self._header["MJD-OBS"], scale="utc", format="mjd")
-
     @cache_translation
     def to_detector_name(self):
         # Docstring will be inherited. Property defined in properties.py
@@ -229,10 +247,14 @@ def to_physical_filter(self):
                         self._log_prefix, default, filter_pos)
         return default
 
+    @cache_translation
     def to_exposure_id(self):
         """Generate a unique exposure ID number
 
-        Note that SEQNUM is not unique for a given day in TS8 data
+        Modern TS8 data conforms to standard LSSTCam OBSID, using the "C"
+        controller variant (all TS8 uses "C" controller).
+
+        For older data SEQNUM is not unique for a given day in TS8 data
         so instead we convert the ISO date of observation directly to an
         integer.
 
@@ -241,10 +263,17 @@ def to_exposure_id(self):
         exposure_id : `int`
             Unique exposure number.
         """
+        obsid = self.to_observation_id()
+        if obsid.startswith("TS_C_"):
+            return super().to_exposure_id()
+
         iso = self._header["DATE-OBS"]
         self._used_these_cards("DATE-OBS")
 
-        return self.compute_exposure_id(iso)
+        # There is worry that seconds are too coarse so use 10th of second
+        # and read the first 21 characters.
+        exposure_id = re.sub(r"\D", "", iso[:21])
+        return int(exposure_id)
 
     # For now assume that visit IDs and exposure IDs are identical
     to_visit_id = to_exposure_id

tests/headers/ts8-TS_C_20230512_000021_R22_S02-fixed.yaml

@@ -0,0 +1,223 @@
+SIMPLE: true
+EXTEND: true
+DATE: '2023-05-12T19:17:59.154'
+MJD: 60076.80415687477
+RUNNUM: '7187D'
+CCD_MANU: 'ITL'
+CCD_TYPE: '3800C'
+TESTTYPE: 'SFLAT_HI'
+IMGTYPE: 'FLAT'
+EXPTIME: 15.0
+FILENAME: 'TS_C_20230512_000021_R22_S02.fits'
+BINX: 1
+BINY: 1
+CCDGAIN: 1.0
+CCDNOISE: 10.0
+DATE-OBS: '2023-05-12T19:17:59.150'
+MJD-OBS: 60076.80415682867
+DATE-TRG: '2023-05-12T19:17:59.150'
+MJD-TRG: 60076.80415682867
+IMAGETAG: 'a5de3d0df4a42e5c'
+CCDSLOT: 'S02'
+RAFTBAY: 'R22'
+FIRMWARE: '3139500e'
+PLATFORM: 'ir2-teststand'
+CONTNUM: '18edf1f0'
+DAQVERS: 'R5-V6.1 2023-04-19T04:26:24Z (1fa91971)'
+DAQPART: 'ts8'
+DAQFOLD: 'raw'
+OBSANNOT: ''
+OBSID: 'TS_C_20230512_000021'
+CAMCODE: 'TS'
+CONTRLLR: 'C'
+DAYOBS: '20230512'
+SEQNUM: 21
+HEADVER: 2
+INSTRUME: 'LSST-TS8'
+TELESCOP: 'LSST'
+TSTAND: 'TS8'
+SEQFILE: 'FP_ITL_2s_ir2_v26.seq'
+SEQCKSUM: '980618532'
+LSST_NUM: 'ITL-3800C-380'
+CCD_SERN: '24350'
+REBNAME: 'LCA-13574-061'
+RAFTNAME: 'LCA-11021_RTM-004'
+DARKTIME: 15.165
+TSEQNUM: 18
+FPVERS: '1.1.8-SNAPSHOT'
+IHVERS: '1.0.36'
+FILTER: ''
+FILTER1: 'HIGH'
+TEMPLED1: 25.495
+TEMPLED2: 25.35
+TEMPBRD: 33.2544
+CCOBLED: 'nm750'
+CCOBCURR: 0.273
+CCOBADC: 1.30239
+CCOBFLST: 1.44006
+PROJTIME: 1.555
+CCOBFLUX: 50000.0
+DATEPBEG: '2023-05-12T19:17:44.058'
+MJDPBEG: 60076.8039821526
+DATEPEND: '2023-05-12T19:17:45.613'
+MJDPEND: 60076.80400015041
+XTENSION: 'BINTABLE'
+BITPIX: 8
+NAXIS: 2
+NAXIS1: 8
+NAXIS2: 2048
+PCOUNT: 1496755
+GCOUNT: 1
+TFIELDS: 1
+TTYPE1: 'COMPRESSED_DATA'
+TFORM1: '1PB(756)'
+ZIMAGE: true
+ZTILE1: 576
+ZTILE2: 1
+ZCMPTYPE: 'RICE_1'
+ZNAME1: 'BLOCKSIZE'
+ZVAL1: 32
+ZNAME2: 'BYTEPIX'
+ZVAL2: 4
+ZTENSION: 'IMAGE'
+ZBITPIX: 32
+ZNAXIS: 2
+ZNAXIS1: 576
+ZNAXIS2: 2048
+ZPCOUNT: 0
+ZGCOUNT: 1
+COMMENT: '---- Checksums ----'
+ZHECKSUM: 'RaeiUaegRaegRaeg'
+CHANNEL: 1
+EXTNAME: 'Segment10'
+CCDSUM: '1 1'
+DATASEC: '[4:512,1:2000]'
+DETSEC: '[509:1,1:2000]'
+DETSIZE: '[1:4072,1:4000]'
+DTV1: 513
+DTV2: 0
+DTM1_1: -1.0
+DTM2_2: 1.0
+DTM1_2: 0.0
+DTM2_1: 0.0
+CTYPE1A: 'Seg_X'
+CTYPE2A: 'Seg_Y'
+CRPIX1A: 0.0
+CRPIX2A: 0.0
+CRVAL1A: 2001.0
+CRVAL2A: 513.0
+BSCALE: 1.0
+BZERO: 0.0
+ZDATASUM: '4028129626'
+CHECKSUM: 'g3KXg2IWg2IWg2IW'
+DATASUM: '3420371118'
+TEMP1: -1.125
+TEMP2: -1.4375
+TEMP3: -14.375
+TEMP4: -9.125
+TEMP5: -11.0625
+TEMP6: -8.1875
+TEMP7: -14.375
+TEMP8: -9.25
+TEMP9: -11.25
+TEMP10: -8.1875
+ATEMPU: -15.6602
+ATEMPL: -13.0444
+CCDTEMP: -100.168
+RTDTEMP: -100.74
+DIGPS_V: 5.025
+DIGPS_I: 744.25
+ANAPS_V: 7.475
+ANAPS_I: 565.0
+CLKHPS_V: 15.3
+CLKHPS_I: 106.0
+CLKLPS_V: 13.2
+CLKLPS_I: 55.125
+ODPS_V: 37.85
+ODPS_I: 90.6667
+HTRPS_V: 3.52819
+HTRPS_W: 0.824041
+PCKU_V: 2.0
+PCKL_V: -8.00733
+SCKU_V: 5.00366
+SCKL_V: -5.00366
+RGU_V: 8.00733
+RGL_V: -2.0
+ODV: 26.8889
+OGV: -2.0
+RDV: 12.9744
+GDV: 19.9853
+GDP: 20.0
+RDP: 13.0
+OGP: -2.0
+ODP: 26.9
+CSGATEP: 1.0
+SCK_LOWP: -5.0
+SCK_HIP: 5.0
+PCK_LOWP: -8.0
+PCK_HIP: 2.0
+RG_LOWP: -2.0
+RG_HIP: 8.0
+AP0_RC: 14
+AP1_RC: 14
+AP0_GAIN: 0
+AP1_GAIN: 0
+AP0_CLMP: 0
+AP1_CLMP: 0
+AP0_AF1: 0
+AP1_AF1: 0
+AP0_TM: 0
+AP1_TM: 0
+HVBIAS: 'ON'
+IDLEFLSH: -1
+POWER: 17.3526
+DIGVB: 5.775
+DIGIB: 771.26
+DIGVA: 5.58272
+DIGIA: 744.268
+DIGVS: 5.5022
+ANAVB: 8.45
+ANAIB: 555.07
+ANAVA: 7.90592
+ANAIA: 545.088
+ANAIS: 7.9544
+ODVB: 40.45
+ODIB: 96.5692
+ODVA: 37.3478
+ODVA2: 38.05
+ODIA: 94.2952
+ODVS: 37.9246
+CKHVB: 15.875
+CKHIB: 101.436
+CKHVA: 15.3729
+CKHIA: 104.076
+CKHVS: 15.426
+CKLVB: 13.925
+CKLIB: 63.8436
+CKLVA: 0.63712
+CKLV2: 13.2879
+CKLIA: 61.7283
+CKLVS: 13.4607
+HTRVB: 11.875
+HTRIB: 230.364
+HTRVA: 11.4712
+HTRIA: 9.912
+HTRVAS: 11.628
+BSSVBS: 49.725
+BSSIBS: 0.078312
+FPDAQCS: '1956679645'
+FPHIDCS: '2732096285'
+FPINSCS: '3879085588'
+FPLIMCS: '152379070'
+FPRTCCS: '3474462408'
+FPRTCSCS: '1974405417'
+FPRCS: '3224983765'
+FPRLCS: '2474308922'
+FPRPCS: '3290985995'
+FPSEQCS: '3477696104'
+FPTIMCS: '2566798867'
+RPLIMCS: '2460191844'
+RPPOWCS: '1782352873'
+RPTIMCS: '3446396476'
+HIERARCH ASTRO METADATA FIX MODIFIED: false
+HIERARCH ASTRO METADATA FIX DATE: '2023-05-19T01:12:26.255716'

tests/headers/ts8-TS_C_20230512_000021_R22_S02.yaml

@@ -1,5 +1,5 @@
-SIMPLE: True
-EXTEND: True
+SIMPLE: true
+EXTEND: true
 DATE: '2023-05-12T19:17:59.154'
 MJD: 60076.80415687477
 RUNNUM: '7187D'
@@ -71,7 +71,7 @@ GCOUNT: 1
 TFIELDS: 1
 TTYPE1: 'COMPRESSED_DATA'
 TFORM1: '1PB(756)'
-ZIMAGE: True
+ZIMAGE: true
 ZTILE1: 576
 ZTILE2: 1
 ZCMPTYPE: 'RICE_1'
@@ -219,5 +219,3 @@ FPTIMCS: '2566798867'
 RPLIMCS: '2460191844'
 RPPOWCS: '1782352873'
 RPTIMCS: '3446396476'
-HIERARCH ASTRO METADATA FIX MODIFIED: False
-HIERARCH ASTRO METADATA FIX DATE: '2023-05-19T01:12:26.255716'