Merge pull request #161 from lsst/tickets/DM-23171

DM-23171: Add exposure_group calculation and make visit_id use it

config/ingest.py

@@ -44,6 +44,7 @@
     'snap': 'translate_snap',
     'controller': 'translate_controller',
     'obsid': 'translate_obsid',
+    'expGroup': 'translate_expGroup',
 }
 config.parse.defaults = {
 }
@@ -67,6 +68,7 @@
     'wavelength': 'int',
     'controller': 'text',
     'obsid': 'text',
+    'expGroup': 'text',
 }
 config.register.unique = ["visit", "detector"]
 config.register.visit = ['visit', 'filter', 'dateObs', 'expTime']

python/lsst/obs/lsst/ingest.py

@@ -277,6 +277,9 @@ def translate_visit(self, md):
     def translate_obsid(self, md):
         return self.observationInfo.observation_id
 
+    def translate_expGroup(self, md):
+        return self.observationInfo.exposure_group
+
     def translate_controller(self, md):
         if "CONTRLLR" in md:
             if md["CONTRLLR"]:

python/lsst/obs/lsst/translators/latiss.py

@@ -222,8 +222,14 @@ def fix_header(cls, header):
                 if not imgType:
                     header["IMGTYPE"] = groupId
                     header["GROUPID"] = None
-                    log.debug("%s: Setting IMGTYPE from GROUPID", obsid)
+                    log.debug("%s: Setting IMGTYPE to '%s' from GROUPID", obsid, header["IMGTYPE"])
                     modified = True
+                else:
+                    # Someone could be fixing headers in old data
+                    # and we do not want GROUPID == IMGTYPE
+                    if imgType == groupId:
+                        # Clear the group so we default to original
+                        header["GROUPID"] = None
 
         # We were using OBJECT for engineering observations early on
         if date < OBJECT_IS_ENGTEST:

python/lsst/obs/lsst/translators/lsst.py

@@ -16,6 +16,9 @@
 import os.path
 import yaml
 import logging
+import re
+import datetime
+import hashlib
 
 import astropy.coordinates
 import astropy.units as u
@@ -30,7 +33,11 @@
 
 # LSST day clock starts at UTC+8
 ROLLOVERTIME = TimeDelta(8*60*60, scale="tai", format="sec")
-TZERO = Time("2010-01-01T00:00", format="isot", scale="utc")
+TZERO = Time("2015-01-01T00:00", format="isot", scale="utc")
+TZERO_DATETIME = TZERO.to_datetime()
+
+# Regex to use for parsing a GROUPID string
+GROUP_RE = re.compile(r"^(\d\d\d\d\-\d\d\-\d\dT\d\d:\d\d:\d\d)\.(\d\d\d)(?:[\+#](\d+))?$")
 
 # LSST Default location in the absence of headers
 LSST_LOCATION = EarthLocation.from_geodetic(-70.749417, -30.244639, 2663.0)
@@ -515,8 +522,62 @@ def to_exposure_id(self):
 
         return self.compute_exposure_id(dayobs, seqnum, controller=controller)
 
-    # For now "visits" are defined to be identical to exposures.
-    to_visit_id = to_exposure_id
+    @cache_translation
+    def to_visit_id(self):
+        """Calculate the visit associated with this exposure.
+
+        Notes
+        -----
+        For LATISS and LSSTCam the default visit is derived from the
+        exposure group.  For other instruments we return the exposure_id.
+        """
+
+        exposure_group = self.to_exposure_group()
+        # If the group is an int we return it
+        try:
+            visit_id = int(exposure_group)
+            return visit_id
+        except ValueError:
+            pass
+
+        # A Group is defined as ISO date with an extension
+        # The integer must be the same for a given group so we can never
+        # use datetime_begin.
+        # Nominally a GROUPID looks like "ISODATE+N" where the +N is
+        # optional.  This can be converted to seconds since epoch with
+        # an adjustment for N.
+        # For early data lacking that form we hash the group and return
+        # the int.
+        matches_date = GROUP_RE.match(exposure_group)
+        if matches_date:
+            iso_str = matches_date.group(1)
+            fraction = matches_date.group(2)
+            n = matches_date.group(3)
+            if n is not None:
+                n = int(n)
+            else:
+                n = 0
+            iso = datetime.datetime.strptime(iso_str, "%Y-%m-%dT%H:%M:%S")
+
+            tdelta = iso - TZERO_DATETIME
+            epoch = int(tdelta.total_seconds())
+
+            # Form the integer from EPOCH + 3 DIGIT FRAC + 0-pad N
+            visit_id = int(f"{epoch}{fraction}{n:04d}")
+        else:
+            # Non-standard string so convert to numbers
+            # using a hash function. Use the first N hex digits
+            group_bytes = exposure_group.encode("us-ascii")
+            hasher = hashlib.blake2b(group_bytes)
+            # Need to be big enough it does not possibly clash with the
+            # date-based version above
+            digest = hasher.hexdigest()[:14]
+            visit_id = int(digest, base=16)
+
+            # To help with hash collision, append the string length
+            visit_id = int(f"{visit_id}{len(exposure_group):02d}")
+
+        return visit_id
 
     @cache_translation
     def to_physical_filter(self):
@@ -561,3 +622,17 @@ def to_altaz_begin(self):
 
         return altaz_from_degree_headers(self, (("ELSTART", "AZSTART"),),
                                          self.to_datetime_begin(), is_zd=False)
+
+    @cache_translation
+    def to_exposure_group(self):
+        """Calculate the exposure group string.
+
+        For LSSTCam and LATISS this is read from the ``GROUPID`` header.
+        If that header is missing the exposure_id is returned instead as
+        a string.
+        """
+        if self.is_key_ok("GROUPID"):
+            exposure_group = self._header["GROUPID"]
+            self._used_these_cards("GROUPID")
+            return exposure_group
+        return super().to_exposure_group()

tests/headers/latiss-future.yaml

@@ -15,7 +15,7 @@ DATE-END: '2020-03-29T16:57:37.055'
 MJD: 58937.7067137073
 MJD-OBS: 58937.7066788773
 OBSID: AT_X_20200329_000022
-GROUPID: 'AGROUPID'
+GROUPID: '2020-03-29T16:55:00.012#24'
 OBSTYPE: 'bias'
 BUNIT: adu
 TELESCOP: LSST AuxTelescope

tests/test_parsetask.py

@@ -125,6 +125,7 @@ def test_parsetask_latiss_translator(self):
                           wavelength=-666,
                           controller="C",
                           obsid="AT_C_20180920_000065",
+                          expGroup="3018092000065",
                       )),
                      )
         self.assertParseCompare(DATADIR, CONFIGDIR, "latiss", LatissParseTask, test_data)
@@ -179,6 +180,7 @@ def test_parsetask_ts8_translator(self):
                           testSeqNum=17,
                           controller="C",
                           obsid="E2V-CCD250-179_lambda_flat_0700_6006D_20180724102845",
+                          expGroup="201807241028453",
                       )),
                      )
         self.assertParseCompare(DATADIR, CONFIGDIR, "ts8", Ts8ParseTask, test_data)
@@ -216,6 +218,7 @@ def test_parsetask_ts3_translator(self):
                           testSeqNum=67,
                           controller="C",
                           obsid="ITL-3800C-098_lambda_flat_1000_067_20160722020740",
+                          expGroup="201607220607067",
                       )),
                      ("raw/2018-11-15/201811151255111-R433-S00-det433.fits",
                       dict(
@@ -237,6 +240,7 @@ def test_parsetask_ts3_translator(self):
                           testSeqNum=25,
                           controller="C",
                           obsid="E2V-CCD250-411_lambda_flat_1000_025_20181115075559",
+                          expGroup="201811151255111",
                       )),
                      )
         self.assertParseCompare(DATADIR, CONFIGDIR, "ts3", Ts3ParseTask, test_data)
@@ -273,6 +277,7 @@ def test_parsetask_imsim_translator(self):
                           snap=0,
                           controller="O",
                           obsid="204595",
+                          expGroup="204595",
                       )),
                      )
         self.assertParseCompare(DATADIR, CONFIGDIR, "imsim", ImsimParseTask, test_data)
@@ -298,6 +303,7 @@ def test_parsetask_phosim_translator(self):
                           visit=204595,
                           controller="O",
                           obsid="204595",
+                          expGroup="204595",
                       )),
                      )
         self.assertParseCompare(DATADIR, CONFIGDIR, "phosim", PhosimParseTask, test_data)
@@ -323,6 +329,7 @@ def test_parsetask_phosim_eimg_translator(self):
                           visit=9006002,
                           controller="O",
                           obsid="9006002",
+                          expGroup="9006002",
                       )),
                      )
         self.assertParseCompare(DATADIR, CONFIGDIR, "phosim", PhosimEimgParseTask, test_data)
@@ -350,6 +357,7 @@ def test_parsetask_ucd_translator(self):
                           testSeqNum=100,
                           controller="C",
                           obsid="E2V-CCD250-112-04_flat_flat_100_20181205153143",
+                          expGroup="20181205233148",
                       )),
                      ("raw/2018-05-30/20180530150355-S00-det002.fits",
                       dict(
@@ -371,6 +379,7 @@ def test_parsetask_ucd_translator(self):
                           testSeqNum=100,
                           controller="C",
                           obsid="ITL-3800C-002_flat_flat_100_20180530080354",
+                          expGroup="20180530150355",
                       )),
                      )
         self.assertParseCompare(DATADIR, CONFIGDIR, "ucd", UcdParseTask, test_data)
@@ -409,6 +418,7 @@ def test_parsetask_lsstCam_translator(self):
                           snap=0,
                           controller="C",
                           obsid="MC_C_20190319_000001",
+                          expGroup="3019031900001",
                       )),
                      ("raw/6489D/R10/3019032200002-R10-S22-det035-000.fits",
                       dict(
@@ -431,6 +441,7 @@ def test_parsetask_lsstCam_translator(self):
                           snap=0,
                           controller="C",
                           obsid="MC_C_20190322_000002",
+                          expGroup="3019032200002",
                       )),
                      )
         self.assertParseCompare(DATADIR, CONFIGDIR, "lsstCam", LsstCamParseTask, test_data)
@@ -458,6 +469,7 @@ def test_parsetask_comCam_translator(self):
                           snap=0,
                           controller="C",
                           obsid="CC_C_20190530_000001",
+                          expGroup="3019053000001",
                       )),
                      )
         self.assertParseCompare(DATADIR, CONFIGDIR, "comCam", LsstComCamParseTask, test_data)