Make target handler configurable

python/lsst/obs/base/gen2to3/calibRepoConverter.py

@@ -25,7 +25,7 @@
 import os
 import sqlite3
 from datetime import datetime, timedelta
-from typing import TYPE_CHECKING, Dict, Iterator, Tuple, Type, Union
+from typing import TYPE_CHECKING, Dict, Iterator, Tuple, Type, Union, Optional
 
 from lsst.daf.butler import Butler as Butler3
 
@@ -35,6 +35,7 @@
 
 if TYPE_CHECKING:
     from lsst.daf.butler import StorageClass, Formatter
+    from .repoWalker.scanner import PathElementHandler
     from ..cameraMapper import CameraMapper
     from ..mapping import Mapping as CameraMapperMapping  # disambiguate from collections.abc.Mapping
 
@@ -66,7 +67,8 @@ def iterMappings(self) -> Iterator[Tuple[str, CameraMapperMapping]]:
         yield from self.mapper.calibrations.items()
 
     def makeRepoWalkerTarget(self, datasetTypeName: str, template: str, keys: Dict[str, type],
-                             storageClass: StorageClass, formatter: Union[None, str, Type[Formatter]] = None
+                             storageClass: StorageClass, formatter: Union[None, str, Type[Formatter]] = None,
+                             targetHandler: Optional[PathElementHandler] = None,
                              ) -> RepoWalker.Target:
         # Docstring inherited from RepoConverter.
         target = RepoWalker.Target(
@@ -77,6 +79,7 @@ def makeRepoWalkerTarget(self, datasetTypeName: str, template: str, keys: Dict[s
             instrument=self.task.instrument.getName(),
             universe=self.task.registry.dimensions,
             formatter=formatter,
+            targetHandler=targetHandler,
         )
         self._datasetTypes.add(target.datasetType)
         return target

python/lsst/obs/base/gen2to3/convertRepo.py

@@ -150,6 +150,12 @@ class ConvertRepoConfig(Config):
         itemtype=str,
         default={}
     )
+    targetHandlerClasses = DictField(
+        "Mapping from dataset type name to target handler class.",
+        keytype=str,
+        itemtype=str,
+        default={}
+    )
     doRegisterInstrument = Field(
         "If True (default), add dimension records for the Instrument and its "
         "filters and detectors to the registry instead of assuming they are "

python/lsst/obs/base/gen2to3/repoConverter.py

@@ -40,13 +40,15 @@
     TYPE_CHECKING,
 )
 
+from lsst.utils import doImport
 from lsst.daf.butler import DataCoordinate, FileDataset, DatasetType
 from lsst.sphgeom import RangeSet, Region
 from .repoWalker import RepoWalker
 
 if TYPE_CHECKING:
     from ..mapping import Mapping as CameraMapperMapping  # disambiguate from collections.abc.Mapping
     from .convertRepo import ConvertRepoTask
+    from .scanner import PathElementHandler
     from lsst.daf.butler import StorageClass, Registry, SkyPixDimension, Formatter
 
 
@@ -253,7 +255,9 @@ def iterMappings(self) -> Iterator[Tuple[str, CameraMapperMapping]]:
     @abstractmethod
     def makeRepoWalkerTarget(self, datasetTypeName: str, template: str, keys: Dict[str, type],
                              storageClass: StorageClass,
-                             formatter: Union[None, str, Type[Formatter]] = None) -> RepoWalker.Target:
+                             formatter: Union[None, str, Type[Formatter]] = None,
+                             targetHandler: Optional[PathElementHandler] = None,
+                             ) -> RepoWalker.Target:
         """Make a struct that identifies a dataset type to be extracted by
         walking the repo directory structure.
 
@@ -269,6 +273,8 @@ def makeRepoWalkerTarget(self, datasetTypeName: str, template: str, keys: Dict[s
             Gen3 storage class for this dataset type.
         formatter : `lsst.daf.butler.Formatter` or `str`, optional
             A Gen 3 formatter class or fully-qualified name.
+        targetHandler : `PathElementHandler`, optional
+            Specialist target handler to use for this dataset type.
 
         Returns
         -------
@@ -356,12 +362,16 @@ class implementation at some point in their own logic.
                     self.task.log.debug("Skipping template in walker: %s", template)
                 else:
                     assert message is None
+                    targetHandler = self.task.config.targetHandlerClasses.get(datasetTypeName)
+                    if targetHandler is not None:
+                        targetHandler = doImport(targetHandler)
                     walkerInput = self.makeRepoWalkerTarget(
                         datasetTypeName=datasetTypeName,
                         template=template+extension,
                         keys=mapping.keys(),
                         storageClass=storageClass,
                         formatter=self.task.config.formatterClasses.get(datasetTypeName),
+                        targetHandler=targetHandler,
                     )
                     self.task.log.debug("Adding template to walker: %s", template)
                 walkerInputs.append(walkerInput)

python/lsst/obs/base/gen2to3/repoWalker/builders.py

@@ -46,7 +46,7 @@
 from .parser import PathElementParser
 from .scanner import PathElementHandler, DirectoryScanner
 from .handlers import (IgnoreHandler, SubdirectoryHandler, SkipHandler,
-                       TargetFileHandler, MultiExtensionFileHandler)
+                       TargetFileHandler)
 
 
 class BuilderNode(ABC):
@@ -196,35 +196,33 @@ class BuilderTargetInput(BuilderInput):
         All candidate dimensions for the Gen3 dataset type.
     formatter : `lsst.daf.butler.Formatter` or `str`, optional
         A Gen 3 formatter class or fully-qualified name.
+    targetHandler : `PathElementHandler`, optional
+        Override target handler for this dataset type.
     kwargs:
         Additional keyword arguments are passed to `Translator.makeMatching`,
         in along with ``datasetTypeName`` and ``keys``.
     """
     def __init__(self, *, datasetTypeName: str, template: str, keys: Dict[str, type],
                  storageClass: StorageClass, universe: DimensionUniverse,
-                 formatter: Union[None, str, Type[Formatter]], **kwargs: Any):
+                 formatter: Union[None, str, Type[Formatter]],
+                 targetHandler: Optional[PathElementHandler] = None, **kwargs: Any):
         # strip off [%HDU] identifiers from e.g. DECAM Community Pipeline products
         template = template.split('[%(')[0]
         super().__init__(template=template, keys=keys)
         self._translator = Translator.makeMatching(datasetTypeName, keys, **kwargs)
         self.datasetType = DatasetType(datasetTypeName, dimensions=self._translator.dimensionNames,
                                        storageClass=storageClass, universe=universe)
         self._formatter = formatter
+        if targetHandler is None:
+            targetHandler = TargetFileHandler
+        self._handler = targetHandler
 
     def build(self, parser: PathElementParser, allKeys: Dict[str, type], cumulativeKeys: Dict[str, type], *,
               fileIgnoreRegEx: Optional[re.Pattern], dirIgnoreRegEx: Optional[re.Pattern]
               ) -> PathElementHandler:
-        # Docstring inherited from BuilderNode.
-        if self.datasetType.name in {'cpBias', 'cpFlat'}:
-            # 'cpBias'/'cpFlat' are DECam Community Pipeline calibrations
-            # stored as multi-extension FITS files.
-            return MultiExtensionFileHandler(parser=parser,
-                                             translator=self._translator,
-                                             datasetType=self.datasetType,
-                                             formatter=self._formatter)
-        else:
-            return TargetFileHandler(parser=parser, translator=self._translator, datasetType=self.datasetType,
-                                     formatter=self._formatter)
+
+        return self.handler(parser=parser, translator=self._translator, datasetType=self.datasetType,
+                            formatter=self._formatter)
 
     def prune(self) -> Tuple[BuilderNode, List[str], bool]:
         # Docstring inherited from BuilderNode.

python/lsst/obs/base/gen2to3/standardRepoConverter.py

@@ -41,6 +41,7 @@
     from lsst.skymap import BaseSkyMap
     from lsst.daf.butler import StorageClass, Formatter
     from .cameraMapper import CameraMapper
+    from .repoWalker.scanner import PathElementHandler
     from ..mapping import Mapping as CameraMapperMapping  # disambiguate from collections.abc.Mapping
 
 
@@ -165,7 +166,9 @@ def findMatchingSkyMap(self, datasetTypeName: str) -> Tuple[Optional[BaseSkyMap]
 
     def makeRepoWalkerTarget(self, datasetTypeName: str, template: str, keys: Dict[str, type],
                              storageClass: StorageClass,
-                             formatter: Union[None, str, Type[Formatter]] = None) -> RepoWalker.Target:
+                             formatter: Union[None, str, Type[Formatter]] = None,
+                             targetHandler: Optional[PathElementHandler] = None,
+                             ) -> RepoWalker.Target:
         # Docstring inherited from RepoConverter.
         skyMap, skyMapName = self.findMatchingSkyMap(datasetTypeName)
         return RepoWalker.Target(
@@ -178,6 +181,7 @@ def makeRepoWalkerTarget(self, datasetTypeName: str, template: str, keys: Dict[s
             skyMap=skyMap,
             skyMapName=skyMapName,
             formatter=formatter,
+            targetHandler=targetHandler,
         )
 
     def iterDatasets(self) -> Iterator[FileDataset]: