Merge pull request #175 from lsst/tickets/DM-20763

DM-20763: Add initial support for Gen3 Butler to obs_decam

python/lsst/daf/butler/instrument.py

@@ -22,9 +22,11 @@
 __all__ = ("Instrument", "updateExposureEntryFromObsInfo", "updateVisitEntryFromObsInfo",
            "ObservationDataIdPacker", "addUnboundedCalibrationLabel")
 
+import os.path
 from datetime import datetime
 from inspect import isabstract
 from abc import ABCMeta, abstractmethod
+
 from lsst.daf.butler import DataId, DataIdPacker
 
 
@@ -47,6 +49,24 @@ class Instrument(metaclass=ABCMeta):
     a no-argument callable that can be used to construct a Python instance.
     """
 
+    configPaths = []
+    """Paths to config files to read for specific Tasks.
+
+    The paths in this list should contain files of the form `task.py`, for
+    each of the Tasks that requires special configuration.
+    """
+
+    @property
+    @abstractmethod
+    def filterDefinitions(self):
+        """`~lsst.obs.base.FilterDefinitionCollection`, defining the filters
+        for this instrument.
+        """
+        return None
+
+    def __init__(self, *args, **kwargs):
+        self.filterDefinitions.defineFilters()
+
     def __init_subclass__(cls, **kwargs):
         super().__init_subclass__(**kwargs)
         if not isabstract(cls):
@@ -58,13 +78,39 @@ def __init_subclass__(cls, **kwargs):
     def getName(cls):
         raise NotImplementedError()
 
+    @abstractmethod
+    def getCamera(self):
+        """Retrieve the cameraGeom representation of this instrument.
+
+        This is a temporary API that should go away once obs_ packages have
+        a standardized approach to writing versioned cameras to a Gen3 repo.
+        """
+        raise NotImplementedError()
+
     @abstractmethod
     def register(self, registry):
         """Insert instrument, physical_filter, and detector entries into a
         `Registry`.
         """
         raise NotImplementedError()
 
+    def _registerFilters(self, registry):
+        """Register the physical and abstract filter Dimension relationships.
+        This should be called in the ``register`` implementation.
+
+        Parameters
+        ----------
+        registry : `lsst.daf.butler.core.Registry`
+            The registry to add dimensions to.
+        """
+        for filter in self.filterDefinitions:
+            registry.addDimensionEntry(
+                "physical_filter",
+                instrument=self.getName(),
+                physical_filter=filter.physical_filter,
+                abstract_filter=filter.abstract_filter
+            )
+
     @abstractmethod
     def getRawFormatter(self, dataId):
         """Return the Formatter class that should be used to read a particular
@@ -93,7 +139,6 @@ def writeCuratedCalibrations(self, butler):
         """
         raise NotImplementedError()
 
-    @abstractmethod
     def applyConfigOverrides(self, name, config):
         """Apply instrument-specific overrides for a task config.
 
@@ -105,7 +150,10 @@ def applyConfigOverrides(self, name, config):
         config : `lsst.pex.config.Config`
             Config instance to which overrides should be applied.
         """
-        raise NotImplementedError()
+        for root in self.configPaths:
+            path = os.path.join(root, f"{name}.py")
+            if os.path.exists(path):
+                config.load(path)
 
 
 class ObservationDataIdPacker(DataIdPacker):

python/lsst/daf/butler/instrument_tests.py

@@ -0,0 +1,95 @@
+# This file is part of daf_butler.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Helpers for writing tests against subclassses of Instrument.
+
+These are not tests themselves, but can be subclassed (plus unittest.TestCase)
+to get a functional test of an Instrument.
+"""
+
+import abc
+import dataclasses
+import os
+
+from lsst.daf.butler import Registry
+from lsst.daf.butler import ButlerConfig
+from lsst.utils import getPackageDir
+
+
+@dataclasses.dataclass
+class InstrumentTestData:
+    """Values to test against in sublcasses of `InstrumentTests`.
+    """
+
+    name: str
+    """The name of the Camera this instrument describes."""
+
+    nDetectors: int
+    """The number of detectors in the Camera."""
+
+    firstDetectorName: str
+    """The name of the first detector in the Camera."""
+
+    physical_filters: {str}
+    """A subset of the physical filters should be registered."""
+
+
+class InstrumentTests(metaclass=abc.ABCMeta):
+    """Tests of sublcasses of Instrument.
+
+    TestCase subclasses must derive from this, then `TestCase`, and override
+    ``data`` and ``instrument``.
+    """
+
+    data = None
+    """`InstrumentTestData` containing the values to test against."""
+
+    instrument = None
+    """The `lsst.daf.butler.Instrument` to be tested."""
+
+    def test_name(self):
+        self.assertEqual(self.instrument.getName(), self.data.name)
+
+    def test_getCamera(self):
+        """Test that getCamera() returns a reasonable Camera definition.
+        """
+        camera = self.instrument.getCamera()
+        self.assertEqual(camera.getName(), self.instrument.getName())
+        self.assertEqual(len(camera), self.data.nDetectors)
+        self.assertEqual(next(iter(camera)).getName(), self.data.firstDetectorName)
+
+    def test_register(self):
+        """Test that register() sets appropriate Dimensions.
+        """
+        registryConfigPath = os.path.join(getPackageDir("daf_butler"), "tests/config/basic/butler.yaml")
+        registry = Registry.fromConfig(ButlerConfig(registryConfigPath))
+        # check that the registry starts out empty
+        self.assertEqual(registry.findDimensionEntries('instrument'), [])
+        self.assertEqual(registry.findDimensionEntries('detector'), [])
+        self.assertEqual(registry.findDimensionEntries('physical_filter'), [])
+
+        # register the instrument and check that certain dimensions appear
+        self.instrument.register(registry)
+        self.assertEqual(len(registry.findDimensionEntries('instrument')), 1)
+        self.assertEqual(registry.findDimensionEntries('instrument')[0]['instrument'], self.data.name)
+        self.assertEqual(len(registry.findDimensionEntries('detector')), self.data.nDetectors)
+        filterNames = {x['physical_filter'] for x in registry.findDimensionEntries('physical_filter')}
+        self.assertGreaterEqual(filterNames, self.data.physical_filters)

tests/test_instrument.py

@@ -33,10 +33,21 @@
 
 class DummyCam(Instrument):
 
+    class FilterDefinitions:
+        """Stopgap until Instrument is moved into obs_base and we can use the
+        real FilterDefinitions there.
+        """
+        def defineFilters(self):
+            pass
+    filterDefinitions = FilterDefinitions()
+
     @classmethod
     def getName(cls):
         return "DummyCam"
 
+    def getCamera(self):
+        return None
+
     def register(self, registry):
         """Insert Instrument, physical_filter, and detector entries into a
         `Registry`.