DM-39346: Fixes for LSSTCam and TS8 filter definitions

python/lsst/obs/lsst/filters.py

@@ -26,6 +26,7 @@
     "TS3_FILTER_DEFINITIONS",
     "TS8_FILTER_DEFINITIONS",
     "COMCAM_FILTER_DEFINITIONS",
+    "GENERIC_FILTER_DEFINITIONS",
 )
 
 from lsst.obs.base import FilterDefinition, FilterDefinitionCollection
@@ -161,6 +162,8 @@ def addFilter(filter_dict, band, physical_filter):
 CCOBFilters = []
 for lsst_filter_def in (EmptyFilter, *LsstCamFiltersBaseline):
     lsstcam_filter = lsst_filter_def.physical_filter
+    if lsstcam_filter == "empty":
+        lsstcam_filter = ""
     lsstcam_band = lsst_filter_def.band
     for ccob_filter, ccob_band in CCOB_filter_map.items():
         if lsstcam_band != "white" and ccob_band != "white" and band != ccob_band:
@@ -336,6 +339,7 @@ def addFilter(filter_dict, band, physical_filter):
                      band="z"),
     FilterDefinition(physical_filter="y_sim_1.4",
                      band="y"),
+    *LsstCamFiltersGeneric,
 )
 
 # ###########################################################################

python/lsst/obs/lsst/translators/lsst.py

@@ -666,14 +666,6 @@ def to_physical_filter(self):
         if not joined:
             joined = "unknown"
 
-        # Remove blank and "empty" fields.
-        joined = FILTER_DELIMITER.join(_ for _ in joined.split(FILTER_DELIMITER)
-                                       if _ and _ != "empty")
-
-        # Return "empty" if joined is blank at this point.
-        if not joined:
-            joined = "empty"
-
         return joined
 
     @cache_translation

python/lsst/obs/lsst/translators/lsstCam.py

@@ -15,6 +15,7 @@
 import logging
 import astropy.units as u
 
+from astro_metadata_translator import cache_translation
 from astro_metadata_translator.translators.helpers import is_non_science
 
 from .lsst import LsstBaseTranslator, SIMONYI_TELESCOPE
@@ -137,3 +138,21 @@ def can_translate(cls, header, filename=None):
             if instrume == cls.supported_instrument.lower():
                 return True
         return False
+
+    @cache_translation
+    def to_physical_filter(self):
+        """Calculate the physical filter name.
+
+        Returns
+        -------
+        filter : `str`
+            Name of filter. Can be a combination of FILTER, FILTER1, and
+            FILTER2 headers joined by a "~".  Trailing "~empty" components
+            are stripped.
+            Returns "unknown" if no filter is declared.
+        """
+        joined = super().to_physical_filter()
+        while joined.endswith("~empty"):
+            joined = joined[:-len("~empty")]
+
+        return joined

tests/headers/ts8-TS_C_20230512_000021_R22_S02-fixed.yaml

@@ -46,7 +46,7 @@ DARKTIME: 15.165
 TSEQNUM: 18
 FPVERS: '1.1.8-SNAPSHOT'
 IHVERS: '1.0.36'
-FILTER: ''
+FILTER: null
 FILTER1: 'HIGH'
 TEMPLED1: 25.495
 TEMPLED2: 25.35

tests/headers/ts8-TS_C_20230512_000021_R22_S02.yaml

@@ -46,7 +46,7 @@ DARKTIME: 15.165
 TSEQNUM: 18
 FPVERS: '1.1.8-SNAPSHOT'
 IHVERS: '1.0.36'
-FILTER: ''
+FILTER: null
 FILTER1: 'HIGH'
 TEMPLED1: 25.495
 TEMPLED2: 25.35

tests/test_filter_defs.py

@@ -0,0 +1,149 @@
+# This file is part of obs_lsst.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import unittest
+from lsst.obs.lsst import (
+    LSSTCAM_FILTER_DEFINITIONS,
+    LATISS_FILTER_DEFINITIONS,
+    LSSTCAM_IMSIM_FILTER_DEFINITIONS,
+    TS3_FILTER_DEFINITIONS,
+    TS8_FILTER_DEFINITIONS,
+    COMCAM_FILTER_DEFINITIONS,
+    GENERIC_FILTER_DEFINITIONS,
+)
+import lsst.obs.lsst.translators  # noqa: F401 -- register the translators
+
+from astro_metadata_translator import ObservationInfo
+from astro_metadata_translator.tests import read_test_file
+
+TESTDIR = os.path.abspath(os.path.dirname(__file__))
+
+
+class FilterDefTestCase(unittest.TestCase):
+    """Each test reads in raw headers from YAML files, constructs an
+    `ObservationInfo`, and checks that the filter definitions for the
+    corresponding instrument contains the `physical_filter` value
+    computed by the translator code.
+    """
+
+    datadir = os.path.join(TESTDIR, "headers")
+
+    def assert_in_filter_defs(self, header_file, filter_def_set):
+        header = read_test_file(header_file, dir=self.datadir)
+        obs_info = ObservationInfo(header, pedantic=True, filename=header_file)
+        self.assertIn(obs_info.physical_filter, filter_def_set)
+
+    def test_lsstCam_filterdefs(self):
+        filter_def_set = set(_.physical_filter for _ in LSSTCAM_FILTER_DEFINITIONS)
+        test_data = (
+            "lsstCam-MC_C_20190319_000001_R10_S02.yaml",
+            "lsstCam-MC_C_20190319_000001_R22_S21.yaml",
+            "lsstCam-MC_C_20190322_000002_R10_S22.yaml",
+            "lsstCam-MC_C_20190406_000643_R10_S00.yaml",
+        )
+        for filename in test_data:
+            with self.subTest(f"Testing {filename}"):
+                self.assert_in_filter_defs(filename, filter_def_set)
+
+    def test_latiss_filterdefs(self):
+        filter_def_set = set(_.physical_filter for _ in LATISS_FILTER_DEFINITIONS)
+        test_data = (
+            "latiss-2018-09-20-05700065-det000.yaml",
+            "latiss-AT_O_20190306_000014.yaml",
+            "latiss-AT_O_20190329_000022-ats-wfs_ccd.yaml",
+            "latiss-AT_O_20190915_000037.yaml",
+            "latiss-AT_O_20191031_000004.yaml",
+            "latiss-AT_O_20191104_000003.yaml",
+            "latiss-AT_O_20191113_000061.yaml",
+            "latiss-AT_O_20200121_000045.yaml",
+            "latiss-AT_O_20200128_000335.yaml",
+            "latiss-AT_O_20200128_000379.yaml",
+            "latiss-AT_O_20210210_000011.yaml",
+            "latiss-AT_O_20210212_000006.yaml",
+            "latiss-AT_O_20220405_000348.yaml",
+            "latiss-AT_O_20220405_000349.yaml",
+            "latiss-AT_O_20230321_000053.yaml",
+            "latiss-future.yaml",
+        )
+        for filename in test_data:
+            with self.subTest(f"Testing {filename}"):
+                self.assert_in_filter_defs(filename, filter_def_set)
+
+    def test_imsim_filterdefs(self):
+        filter_def_set = set(
+            _.physical_filter for _ in LSSTCAM_IMSIM_FILTER_DEFINITIONS
+        )
+        test_data = (
+            "imsim-bias-lsst_a_3010002_R11_S00.yaml",
+            "imsim-dark-lsst_a_4010003_R11_S11.yaml",
+            "imsim-flats-lsst_a_5000007_R11_S20_i.yaml",
+            "imsim-lsst_a_204595_R11_S02_i.yaml",
+        )
+        for filename in test_data:
+            with self.subTest(f"Testing {filename}"):
+                self.assert_in_filter_defs(filename, filter_def_set)
+
+    def test_ts3_filterdefs(self):
+        filter_def_set = set(_.physical_filter for _ in TS3_FILTER_DEFINITIONS)
+        test_data = (
+            "ts3-E2V-CCD250-411_lambda_flat_1000_025_20181115075559.yaml",
+            "ts3-ITL-3800C-098_lambda_flat_1000_067_20160722020740.yaml",
+        )
+        for filename in test_data:
+            with self.subTest(f"Testing {filename}"):
+                self.assert_in_filter_defs(filename, filter_def_set)
+
+    def test_ts8_filterdefs(self):
+        filter_def_set = set(_.physical_filter for _ in TS8_FILTER_DEFINITIONS)
+        test_data = (
+            "ts8-E2V-CCD250-179_lambda_bias_024_6006D_20180724104156.yaml",
+            "ts8-E2V-CCD250-200-Dev_lambda_flat_0700_6006D_20180724102845.yaml",
+            "ts8-E2V-CCD250-220_fe55_fe55_094_6288_20171215114006.yaml",
+            "ts8-TS_C_20220711_000174_R22_S00.yaml",
+            "ts8-TS_C_20230512_000021_R22_S02.yaml",
+        )
+        for filename in test_data:
+            with self.subTest(f"Testing {filename}"):
+                self.assert_in_filter_defs(filename, filter_def_set)
+
+    def test_comCam_filterdefs(self):
+        filter_def_set = set(_.physical_filter for _ in COMCAM_FILTER_DEFINITIONS)
+        test_data = (
+            "comCam-CC_C_20190526_000223_R22_S01.yaml",
+            "comCam-CC_C_20190530_000001_R22_S00.yaml",
+            "comCam-CC_H_20100217_006001_R22_S00.yaml",
+        )
+        for filename in test_data:
+            with self.subTest(f"Testing {filename}"):
+                self.assert_in_filter_defs(filename, filter_def_set)
+
+    def test_generic_filterdefs(self):
+        filter_def_set = set(_.physical_filter for _ in GENERIC_FILTER_DEFINITIONS)
+        test_data = (
+            "phosim-lsst_a_204595_f3_R11_S02_E000.yaml",
+            "lsstCam-MC_H_20100217_000032_R22_S00.yaml",  # This is a phosim header
+            "UCD-E2V-CCD250-112-04_flat_flat_100_20181205153143.yaml",
+            "UCD-ITL-3800C-002_flat_flat_100_20180530080354.yaml",
+        )
+        for filename in test_data:
+            with self.subTest(f"Testing {filename}"):
+                self.assert_in_filter_defs(filename, filter_def_set)