Merge pull request #291 from lsst/tickets/DM-25156

DM-25156: During 2to3 conversion ask Instrument for the curated calibrations

python/lsst/obs/base/_instrument.py

@@ -25,8 +25,9 @@
 
 import os.path
 from abc import ABCMeta, abstractmethod
-from typing import Any, Tuple, TYPE_CHECKING
+from typing import Any, Optional, Set, Sequence, Tuple, TYPE_CHECKING
 import astropy.time
+from functools import lru_cache
 
 from lsst.afw.cameraGeom import Camera
 from lsst.daf.butler import (
@@ -62,31 +63,41 @@ class Instrument(metaclass=ABCMeta):
     arguments.
     """
 
-    configPaths = ()
+    configPaths: Sequence[str] = ()
     """Paths to config files to read for specific Tasks.
 
     The paths in this list should contain files of the form `task.py`, for
     each of the Tasks that requires special configuration.
     """
 
-    policyName = None
+    policyName: Optional[str] = None
     """Instrument specific name to use when locating a policy or configuration
     file in the file system."""
 
-    obsDataPackage = None
+    obsDataPackage: Optional[str] = None
     """Name of the package containing the text curated calibration files.
     Usually a obs _data package.  If `None` no curated calibration files
     will be read. (`str`)"""
 
-    standardCuratedDatasetTypes = tuple(StandardCuratedCalibrationDatasetTypes)
+    standardCuratedDatasetTypes: Set[str] = frozenset(StandardCuratedCalibrationDatasetTypes)
     """The dataset types expected to be obtained from the obsDataPackage.
+
     These dataset types are all required to have standard definitions and
     must be known to the base class.  Clearing this list will prevent
     any of these calibrations from being stored. If a dataset type is not
     known to a specific instrument it can still be included in this list
-    since the data package is the source of truth.
+    since the data package is the source of truth. (`set` of `str`)
     """
 
+    additionalCuratedDatasetTypes: Set[str] = frozenset()
+    """Curated dataset types specific to this particular instrument that do
+    not follow the standard organization found in obs data packages.
+
+    These are the instrument-specific dataset types written by
+    `writeAdditionalCuratedCalibrations` in addition to the calibrations
+    found in obs data packages that follow the standard scheme.
+    (`set` of `str`)"""
+
     @property
     @abstractmethod
     def filterDefinitions(self):
@@ -98,7 +109,6 @@ def filterDefinitions(self):
     def __init__(self):
         self.filterDefinitions.reset()
         self.filterDefinitions.defineFilters()
-        self._obsDataPackageDir = None
 
     @classmethod
     @abstractmethod
@@ -111,6 +121,39 @@ def getName(cls):
         """
         raise NotImplementedError()
 
+    @classmethod
+    @lru_cache()
+    def getCuratedCalibrationNames(cls) -> Set[str]:
+        """Return the names of all the curated calibration dataset types.
+
+        Returns
+        -------
+        names : `set` of `str`
+            The dataset type names of all curated calibrations. This will
+            include the standard curated calibrations even if the particular
+            instrument does not support them.
+
+        Notes
+        -----
+        The returned list does not indicate whether a particular dataset
+        is present in the Butler repository, simply that these are the
+        dataset types that are handled by ``writeCuratedCalibrations``.
+        """
+
+        # Camera is a special dataset type that is also handled as a
+        # curated calibration.
+        curated = {"camera"}
+
+        # Make a cursory attempt to filter out curated dataset types
+        # that are not present for this instrument
+        for datasetTypeName in cls.standardCuratedDatasetTypes:
+            calibPath = cls._getSpecificCuratedCalibrationPath(datasetTypeName)
+            if calibPath is not None:
+                curated.add(datasetTypeName)
+
+        curated.update(cls.additionalCuratedDatasetTypes)
+        return frozenset(curated)
+
     @abstractmethod
     def getCamera(self):
         """Retrieve the cameraGeom representation of this instrument.
@@ -127,18 +170,20 @@ def register(self, registry):
         """
         raise NotImplementedError()
 
-    @property
-    def obsDataPackageDir(self):
-        """The root of the obs package that provides specializations for
-        this instrument (`str`).
+    @classmethod
+    @lru_cache()
+    def getObsDataPackageDir(cls):
+        """The root of the obs data package that provides specializations for
+        this instrument.
+
+        returns
+        -------
+        dir : `str`
+            The root of the relevat obs data package.
         """
-        if self.obsDataPackage is None:
+        if cls.obsDataPackage is None:
             return None
-        if self._obsDataPackageDir is None:
-            # Defer any problems with locating the package until
-            # we need to find it.
-            self._obsDataPackageDir = getPackageDir(self.obsDataPackage)
-        return self._obsDataPackageDir
+        return getPackageDir(cls.obsDataPackage)
 
     @staticmethod
     def fromName(name: str, registry: Registry) -> Instrument:
@@ -349,6 +394,34 @@ def writeStandardTextCuratedCalibrations(self, butler, run=None):
                                       **definition)
             self._writeSpecificCuratedCalibrationDatasets(butler, datasetType, run=run)
 
+    @classmethod
+    def _getSpecificCuratedCalibrationPath(cls, datasetTypeName):
+        """Return the path of the curated calibration directory.
+
+        Parameters
+        ----------
+        datasetTypeName : `str`
+            The name of the standard dataset type to find.
+
+        Returns
+        -------
+        path : `str`
+            The path to the standard curated data directory.  `None` if the
+            dataset type is not found or the obs data package is not
+            available.
+        """
+        if cls.getObsDataPackageDir() is None:
+            # if there is no data package then there can't be datasets
+            return None
+
+        calibPath = os.path.join(cls.getObsDataPackageDir(), cls.policyName,
+                                 datasetTypeName)
+
+        if os.path.exists(calibPath):
+            return calibPath
+
+        return None
+
     def _writeSpecificCuratedCalibrationDatasets(self, butler, datasetType, run=None):
         """Write standardized curated calibration datasets for this specific
         dataset type from an obs data package.
@@ -373,14 +446,8 @@ class attribute for curated calibrations corresponding to the
         `~lsst.pipe.tasks.read_curated_calibs.read_all` and provide standard
         metadata.
         """
-        if self.obsDataPackageDir is None:
-            # if there is no data package then there can't be datasets
-            return
-
-        calibPath = os.path.join(self.obsDataPackageDir, self.policyName,
-                                 datasetType.name)
-
-        if not os.path.exists(calibPath):
+        calibPath = self._getSpecificCuratedCalibrationPath(datasetType.name)
+        if calibPath is None:
             return
 
         # Register the dataset type

python/lsst/obs/base/gen2to3/calibRepoConverter.py

@@ -58,7 +58,7 @@ def __init__(self, *, mapper: CameraMapper, **kwds):
 
     def isDatasetTypeSpecial(self, datasetTypeName: str) -> bool:
         # Docstring inherited from RepoConverter.
-        return datasetTypeName in self.task.config.curatedCalibrations
+        return datasetTypeName in self.instrument.getCuratedCalibrationNames()
 
     def iterMappings(self) -> Iterator[Tuple[str, CameraMapperMapping]]:
         # Docstring inherited from RepoConverter.

python/lsst/obs/base/gen2to3/convertRepo.py

@@ -250,18 +250,6 @@ class ConvertRepoConfig(Config):
         dtype=bool,
         default=False,
     )
-    curatedCalibrations = ListField(
-        "Dataset types that are handled by `Instrument.writeCuratedCalibrations()` "
-        "and thus should not be converted using the standard calibration "
-        "conversion system.",
-        dtype=str,
-        default=["camera",
-                 "transmission_sensor",
-                 "transmission_filter",
-                 "transmission_optics",
-                 "transmission_atmosphere",
-                 "bfKernel"]
-    )
 
     @property
     def transfer(self):
@@ -482,12 +470,12 @@ def run(self, root: str, *,
 
         # Make converters for all Gen2 repos.
         converters = []
-        rootConverter = RootRepoConverter(task=self, root=root, subset=subset)
+        rootConverter = RootRepoConverter(task=self, root=root, subset=subset, instrument=self.instrument)
         converters.append(rootConverter)
         for calibRoot, run in calibs.items():
             if not os.path.isabs(calibRoot):
                 calibRoot = os.path.join(rootConverter.root, calibRoot)
-            converter = CalibRepoConverter(task=self, root=calibRoot, run=run,
+            converter = CalibRepoConverter(task=self, root=calibRoot, run=run, instrument=self.instrument,
                                            mapper=rootConverter.mapper,
                                            subset=rootConverter.subset)
             converters.append(converter)
@@ -496,7 +484,7 @@ def run(self, root: str, *,
             if not os.path.isabs(runRoot):
                 runRoot = os.path.join(rootConverter.root, runRoot)
             converter = StandardRepoConverter(task=self, root=runRoot, run=spec.runName,
-                                              subset=rootConverter.subset)
+                                              instrument=self.instrument, subset=rootConverter.subset)
             converters.append(converter)
 
         # Register the instrument if we're configured to do so.

python/lsst/obs/base/gen2to3/repoConverter.py

@@ -50,6 +50,7 @@
     from .convertRepo import ConvertRepoTask
     from .scanner import PathElementHandler
     from lsst.daf.butler import StorageClass, Registry, SkyPixDimension, FormatterParameter
+    from .._instrument import Instrument
 
 
 @dataclass
@@ -187,6 +188,8 @@ class RepoConverter(ABC):
     root : `str`
         Root of the Gen2 repo being converted.  Will be converted to an
         absolute path, resolving symbolic links and ``~``, if necessary.
+    instrument : `Instrument`
+        Gen3 instrument class to use for this conversion.
     collections : `list` of `str`
         Gen3 collections with which all converted datasets should be
         associated.
@@ -204,10 +207,11 @@ class RepoConverter(ABC):
     implementation.
     """
 
-    def __init__(self, *, task: ConvertRepoTask, root: str, run: Optional[str],
+    def __init__(self, *, task: ConvertRepoTask, root: str, instrument: Instrument, run: Optional[str],
                  subset: Optional[ConversionSubset] = None):
         self.task = task
         self.root = os.path.realpath(os.path.expanduser(root))
+        self.instrument = instrument
         self.subset = subset
         self._run = run
         self._repoWalker = None  # Created in prep

python/lsst/obs/base/gen2to3/rootRepoConverter.py

@@ -88,7 +88,7 @@ def isDatasetTypeSpecial(self, datasetTypeName: str) -> bool:
             super().isDatasetTypeSpecial(datasetTypeName)
             or datasetTypeName in ("raw", "ref_cat", "ref_cat_config")
             # in Gen2, some of these are in the root repo, not a calib repo
-            or datasetTypeName in self.task.config.curatedCalibrations
+            or datasetTypeName in self.instrument.getCuratedCalibrationNames()
         )
 
     def getSpecialDirectories(self) -> List[str]: