Skip to content

Commit

Permalink
Fix TS8 exposure id calculation for historical data
Browse files Browse the repository at this point in the history 
We have already ingested TS8 data into a butler using the old
timestamp exposure_id calculation. It is very disruptive to change
this because it will require the entire repository be remade from
scratch. Furthermore, the fix to DATE-OBS broke all the historical
calculations where the exposure_id was being derived from the end
of the integration by mistake.

This commit retains LSSTCam exposure_id for > 2023-05-24 and
uses the original DATE-OBS value to calculate the exposure ID for
older data.
  • Loading branch information
@timj
timj committed May 25, 2023
1 parent 5f2fb88 commit 5607db3
Show file tree
Hide file tree
Showing 3 changed files with 260 additions and 21 deletions.
22 changes: 17 additions & 5 deletions python/lsst/obs/lsst/translators/ts8.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@

log = logging.getLogger(__name__)

_EXPOSURE_ID_DATE_CHANGE = Time("2023-05-24T12:00:00", format="isot", scale="utc")
_UNMODIFIED_DATE_OBS_HEADER = "HIERARCH LSST-TS8 DATE-OBS"


class LsstTS8Translator(LsstBaseTranslator):
"""Metadata translator for LSST Test Stand 8 data.
Expand Down Expand Up @@ -134,6 +137,10 @@ def fix_header(cls, header, instrument, obsid, filename=None):
break

if date_obs:
# The historical exposure ID calculation requires that we
# have access to the unmodified DATE-OBS value.
header[_UNMODIFIED_DATE_OBS_HEADER] = header["DATE-OBS"]

exptime = TimeDelta(header["EXPTIME"]*u.s, scale="tai")
date_obs = date_obs - exptime
header["MJD-OBS"] = float(date_obs.mjd)
Expand Down Expand Up @@ -252,7 +259,9 @@ def to_exposure_id(self):
"""Generate a unique exposure ID number
Modern TS8 data conforms to standard LSSTCam OBSID, using the "C"
controller variant (all TS8 uses "C" controller).
controller variant (all TS8 uses "C" controller). Due to existing
ingests, data taken before 2023-05-24 must use the old style
timestamp ID.
For older data SEQNUM is not unique for a given day in TS8 data
so instead we convert the ISO date of observation directly to an
Expand All @@ -263,11 +272,14 @@ def to_exposure_id(self):
exposure_id : `int`
Unique exposure number.
"""
obsid = self.to_observation_id()
if obsid.startswith("TS_C_"):
return super().to_exposure_id()
begin = self.to_datetime_begin()

if begin > _EXPOSURE_ID_DATE_CHANGE:
obsid = self.to_observation_id()
if obsid.startswith("TS_C_"):
return super().to_exposure_id()

iso = self._header["DATE-OBS"]
iso = self._header.get(_UNMODIFIED_DATE_OBS_HEADER, self._header["DATE-OBS"])
self._used_these_cards("DATE-OBS")

# There is worry that seconds are too coarse so use 10th of second
Expand Down
203 changes: 203 additions & 0 deletions tests/headers/ts8-TS_C_20230525_000041_R22_S00.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,203 @@
SIMPLE: true
EXTEND: true
DATE: '2023-05-25T17:45:49.870'
MJD: 60089.740160532296
RUNNUM: 7227D
CCD_MANU: ITL
CCD_TYPE: 3800C
TESTTYPE: FLAT
IMGTYPE: FLAT
EXPTIME: 30.0
FILENAME: TS_C_20230525_000041_R22_S00.fits
BINX: 1
BINY: 1
CCDGAIN: 1.0
CCDNOISE: 10.0
DATE-OBS: '2023-05-25T17:45:19.715'
MJD-OBS: 60089.73981151637
DATE-TRG: '2023-05-25T17:45:49.866'
MJD-TRG: 60089.740160486195
IMAGETAG: c71b061eb369b82a
CCDSLOT: S00
RAFTBAY: R22
FIRMWARE: 3139500e
PLATFORM: ir2-teststand
CONTNUM: 18edf1f0
DAQVERS: R5-V6.2 2023-05-17T21:16:12Z (39e4a179)
DAQPART: ts8
DAQFOLD: raw
OBSANNOT: Just R22
OBSID: TS_C_20230525_000041
CAMCODE: TS
CONTRLLR: C
DAYOBS: '20230525'
SEQNUM: 41
HEADVER: 2
INSTRUME: LSST-TS8
TELESCOP: LSST
TSTAND: TS8
SEQFILE: FP_ITL_2s_ir2_v26.seq
SEQCKSUM: '1606982964'
LSST_NUM: ITL-3800C-372
CCD_SERN: '24809'
REBNAME: LCA-13574-061
RAFTNAME: LCA-11021_RTM-004
DARKTIME: 30.151
TSEQNUM: 100
FPVERS: 1.1.8-dev-SNAPSHOT
IHVERS: 1.1.0-dev-SNAPSHOT
DATE-BEG: '2023-05-25T17:45:19.715'
MJD-BEG: 60089.73981151637
DATE-END: '2023-05-25T17:45:49.866'
MJD-END: 60089.740160486195
TIMESYS: TAI
FILTER: null
FILTER1: null
TEMPLED1: 32.4309
TEMPLED2: 32.3017
TEMPBRD: 39.7695
CCOBLED: uv
CCOBCURR: 0.1
CCOBADC: 0.179753
CCOBFLST: 0.701545
PROJTIME: 0.843
CCOBFLUX: 1321.0
DATEPBEG: '2023-05-25T17:45:19.788'
MJDPBEG: 60089.73981236108
DATEPEND: '2023-05-25T17:45:20.631'
MJDPEND: 60089.73982211808
XTENSION: BINTABLE
BITPIX: 8
NAXIS: 2
NAXIS1: 8
NAXIS2: 2048
PCOUNT: 1098676
GCOUNT: 1
TFIELDS: 1
TTYPE1: COMPRESSED_DATA
TFORM1: 1PB(557)
ZIMAGE: true
ZTILE1: 576
ZTILE2: 1
ZCMPTYPE: RICE_1
ZNAME1: BLOCKSIZE
ZVAL1: 32
ZNAME2: BYTEPIX
ZVAL2: 4
ZTENSION: IMAGE
ZBITPIX: 32
ZNAXIS: 2
ZNAXIS1: 576
ZNAXIS2: 2048
ZPCOUNT: 0
ZGCOUNT: 1
COMMENT: '---- Checksums ----'
ZHECKSUM: biK5egK5bgK5bgK5
CHANNEL: 1
EXTNAME: Segment10
CCDSUM: 1 1
DATASEC: '[4:512,1:2000]'
DETSEC: '[509:1,1:2000]'
DETSIZE: '[1:4072,1:4000]'
DTV1: 513
DTV2: 0
DTM1_1: -1.0
DTM2_2: 1.0
DTM1_2: 0.0
DTM2_1: 0.0
WCSNAMEA: AMPLIFIER
CTYPE1A: Seg_X
CTYPE2A: Seg_Y
PC1_1A: 0.0
PC1_2A: -1.0
PC2_1A: -1.0
PC2_2A: 0.0
CDELT1A: 1.0
CDELT2A: 1.0
CRPIX1A: 0.0
CRPIX2A: 0.0
CRVAL1A: 2001.0
CRVAL2A: 513.0
WCSNAMEC: CCD
CTYPE1C: CCD_X
CTYPE2C: CCD_Y
PC1_1C: 0.0
PC1_2C: -1.0
PC2_1C: -1.0
PC2_2C: 0.0
CDELT1C: 1.0
CDELT2C: 1.0
CRPIX1C: 0.0
CRPIX2C: 0.0
CRVAL1C: 4001.0
CRVAL2C: 513.0
WCSNAMER: RAFT
CTYPE1R: RAFT_X
CTYPE2R: RAFT_Y
PC1_1R: 0.0
PC1_2R: -1.0
PC2_1R: -1.0
PC2_2R: 0.0
CDELT1R: 1.0
CDELT2R: 1.0
CRPIX1R: 0.0
CRPIX2R: 0.0
CRVAL1R: 4126.0
CRVAL2R: 602.0
WCSNAMEF: FOCAL_PLANE
CTYPE1F: FP_X
CTYPE2F: FP_Y
PC1_1F: 0.0
PC1_2F: -1.0
PC2_1F: -1.0
PC2_2F: 0.0
CDELT1F: 1.0
CDELT2F: 1.0
CRPIX1F: 0.0
CRPIX2F: 0.0
CRVAL1F: 29526.0
CRVAL2F: 26002.0
WCSNAMEE: FP_SERPAR
CTYPE1E: FP_S
CTYPE2E: FP_P
PC1_1E: -1.0
PC1_2E: 0.0
PC2_1E: 0.0
PC2_2E: -1.0
CDELT1E: 1.0
CDELT2E: 1.0
CRPIX1E: 0.0
CRPIX2E: 0.0
CRVAL1E: 26002.0
CRVAL2E: 29526.0
WCSNAMEB: CCD_SERPAR
CTYPE1B: CCD_S
CTYPE2B: CCD_P
PC1_1B: -1.0
PC1_2B: 0.0
PC2_1B: 0.0
PC2_2B: -1.0
CDELT1B: 1.0
CDELT2B: 1.0
CRPIX1B: 0.0
CRPIX2B: 0.0
CRVAL1B: 513.0
CRVAL2B: 4001.0
WCSNAMEQ: RAFT_SERPAR
CTYPE1Q: RAFT_S
CTYPE2Q: RAFT_P
PC1_1Q: -1.0
PC1_2Q: 0.0
PC2_1Q: 0.0
PC2_2Q: -1.0
CDELT1Q: 1.0
CDELT2Q: 1.0
CRPIX1Q: 0.0
CRPIX2Q: 0.0
CRVAL1Q: 602.0
CRVAL2Q: 4126.0
BSCALE: 1.0
BZERO: 0.0
ZDATASUM: '226966295'
CHECKSUM: SammSYkjSakjSWkj
DATASUM: '1153382584'
56 changes: 40 additions & 16 deletions tests/test_translator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1049,13 +1049,13 @@ def test_ts8_translator(self):
dict(telescope=None,
instrument="LSST-TS8",
dark_time=210.315*u.s,
detector_exposure_id=3022071100174018,
detector_exposure_id=202207111714459018,
detector_group="RTM-004",
detector_name="S00",
detector_num=18,
detector_serial="ITL-3800C-372",
exposure_id=3022071100174,
exposure_group="3022071100174",
exposure_id=202207111714459,
exposure_group="202207111714459",
exposure_time=210.0*u.s,
focus_z=0.0*u.mm,
group_counter_end=174,
Expand All @@ -1068,18 +1068,18 @@ def test_ts8_translator(self):
observing_day=20220711,
physical_filter="unknown",
science_program="7074D",
visit_id=3022071100174)),
visit_id=202207111714459)),
("ts8-TS_C_20230512_000021_R22_S02.yaml",
dict(telescope=None,
instrument="LSST-TS8",
dark_time=15.165*u.s,
detector_exposure_id=3023051200021020,
detector_exposure_id=202305121917591020,
detector_group="RTM-004",
detector_name="S02",
detector_num=20,
detector_serial="ITL-3800C-380",
exposure_id=3023051200021,
exposure_group="3023051200021",
exposure_id=202305121917591,
exposure_group="202305121917591",
exposure_time=15.0*u.s,
focus_z=0.0*u.mm,
group_counter_end=21,
Expand All @@ -1092,18 +1092,18 @@ def test_ts8_translator(self):
observing_day=20230512,
physical_filter="HIGH",
science_program="7187D",
visit_id=3023051200021)),
visit_id=202305121917591)),
("ts8-TS_C_20230512_000021_R22_S02-fixed.yaml", # Will not apply header fixups
dict(telescope=None,
instrument="LSST-TS8",
dark_time=15.165*u.s,
detector_exposure_id=3023051200021020,
detector_exposure_id=202305121917591020,
detector_group="RTM-004",
detector_name="S02",
detector_num=20,
detector_serial="ITL-3800C-380",
exposure_id=3023051200021,
exposure_group="3023051200021",
exposure_id=202305121917591,
exposure_group="202305121917591",
exposure_time=15.0*u.s,
focus_z=0.0*u.mm,
group_counter_end=21,
Expand All @@ -1116,18 +1116,18 @@ def test_ts8_translator(self):
observing_day=20230512,
physical_filter="HIGH",
science_program="7187D",
visit_id=3023051200021)),
visit_id=202305121917591)),
("ts8-TS_C_20230523_000127_R22_S00.yaml",
dict(telescope=None,
instrument="LSST-TS8",
dark_time=15.16*u.s,
detector_exposure_id=3023052300127018,
detector_exposure_id=202305231936194018,
detector_group="RTM-004",
detector_name="S00",
detector_num=18,
detector_serial="ITL-3800C-372",
exposure_id=3023052300127,
exposure_group="3023052300127",
exposure_id=202305231936194,
exposure_group="202305231936194",
exposure_time=15.0*u.s,
focus_z=0.0*u.mm,
group_counter_end=127,
Expand All @@ -1140,7 +1140,31 @@ def test_ts8_translator(self):
observing_day=20230523,
physical_filter="unknown",
science_program="7220D",
visit_id=3023052300127)),
visit_id=202305231936194)),
("ts8-TS_C_20230525_000041_R22_S00.yaml",
dict(telescope=None,
instrument="LSST-TS8",
dark_time=30.151*u.s,
detector_exposure_id=3023052500041018,
detector_group="RTM-004",
detector_name="S00",
detector_num=18,
detector_serial="ITL-3800C-372",
exposure_id=3023052500041,
exposure_group="3023052500041",
exposure_time=30.0*u.s,
focus_z=0.0*u.mm,
group_counter_end=41,
group_counter_start=41,
has_simulated_content=False,
observation_counter=41,
observation_id="TS_C_20230525_000041",
observation_type="flat",
observation_reason="flat",
observing_day=20230525,
physical_filter="unknown",
science_program="7227D",
visit_id=3023052500041)),
)
for filename, expected in test_data:
with self.subTest(f"Testing {filename}"):
Expand Down

0 comments on commit 5607db3

Please sign in to comment.