DM-26119: Changes to support composite disassembly in ci_hsc_gen3

python/lsst/daf/butler/configs/datastores/composites.yaml

@@ -8,6 +8,6 @@ default: false
 # Types can be StorageClass names or DatasetType names, with DatasetType
 # name always taking preference over StorageClass name.
 disassembled:
-  ExposureI: false
-  ExposureF: false
-  calexp: false
+  # This is a place holder to seed the mapping so that specific configs
+  # can override and allows this default to be used as documentation
+  __null__: false

python/lsst/daf/butler/configs/datastores/formatters.yaml

@@ -26,11 +26,13 @@ DataFrame: lsst.daf.butler.formatters.parquet.ParquetFormatter
 Defects: lsst.obs.base.formatters.fitsGeneric.FitsGenericFormatter
 QECurve: lsst.obs.base.formatters.fitsGeneric.FitsGenericFormatter
 CrosstalkCalib: lsst.obs.base.formatters.fitsGeneric.FitsGenericFormatter
-ImageF: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
-ImageU: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
+Image: lsst.obs.base.formatters.fitsExposure.FitsImageFormatter
+ImageF: lsst.obs.base.formatters.fitsExposure.FitsImageFormatter
+ImageU: lsst.obs.base.formatters.fitsExposure.FitsImageFormatter
 DecoratedImageU: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
-MaskX: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
-MaskedImageF: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
+Mask: lsst.obs.base.formatters.fitsExposure.FitsMaskFormatter
+MaskX: lsst.obs.base.formatters.fitsExposure.FitsMaskFormatter
+MaskedImageF: lsst.obs.base.formatters.fitsMaskedImage.FitsMaskedImageFormatter
 Exposure: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
 ExposureF: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter
 ExposureI: lsst.obs.base.formatters.fitsExposure.FitsExposureFormatter

python/lsst/daf/butler/core/composites.py

@@ -83,9 +83,18 @@ def __init__(self, config: Union[str, ButlerConfig, CompositesConfig], *,
         assert isinstance(config, CompositesConfig)
         self.config = config
 
+        # Pre-filter the disassembly lookup table to remove the
+        # placeholder __ key we added for documentation.
+        # It should be harmless but might confuse validation
+        # Retain the entry as a Config so change in place
+        disassemblyMap = self.config[DISASSEMBLY_KEY]
+        for k in set(disassemblyMap):
+            if k.startswith("__"):
+                del disassemblyMap[k]
+
         # Calculate the disassembly lookup table -- no need to process
         # the values
-        self._lut = processLookupConfigs(self.config[DISASSEMBLY_KEY], universe=universe)
+        self._lut = processLookupConfigs(disassemblyMap, universe=universe)
 
     def shouldBeDisassembled(self, entity: Union[DatasetRef, DatasetType, StorageClass]) -> bool:
         """Given some choices, indicate whether the entity should be

python/lsst/daf/butler/core/datasets/ref.py

@@ -133,7 +133,7 @@ def __repr__(self) -> str:
             return f"DatasetRef({self.datasetType!r}, {self.dataId!s})"
 
     def __str__(self) -> str:
-        s = f"{self.datasetType.name}@{self.dataId!s}"
+        s = f"{self.datasetType.name}@{self.dataId!s}, sc={self.datasetType.storageClass.name}]"
         if self.id is not None:
             s += f" (id={self.id})"
         return s

python/lsst/daf/butler/datastores/fileLikeDatastore.py

@@ -115,9 +115,12 @@ class DatastoreFileGetInformation:
     info: StoredFileInfo
     """Stored information about this file and its formatter."""
 
-    assemblerParams: dict
+    assemblerParams: Dict[str, Any]
     """Parameters to use for post-processing the retrieved dataset."""
 
+    formatterParams: Dict[str, Any]
+    """Parameters that were understood by the associated formatter."""
+
     component: Optional[str]
     """The component to be retrieved (can be `None`)."""
 
@@ -474,7 +477,7 @@ def _prepare_for_get(self, ref: DatasetRef,
                                                      storageClass=writeStorageClass, parameters=parameters),
                                       ref.dataId)
 
-            _, notFormatterParams = formatter.segregateParameters()
+            formatterParams, notFormatterParams = formatter.segregateParameters()
 
             # Of the remaining parameters, extract the ones supported by
             # this StorageClass (for components not all will be handled)
@@ -486,7 +489,8 @@ def _prepare_for_get(self, ref: DatasetRef,
             component = storedFileInfo.component if storedFileInfo.component else refComponent
 
             fileGetInfo.append(DatastoreFileGetInformation(location, formatter, storedFileInfo,
-                                                           assemblerParams, component, readStorageClass))
+                                                           assemblerParams, formatterParams,
+                                                           component, readStorageClass))
 
         return fileGetInfo
 
@@ -933,12 +937,17 @@ def get(self, ref: DatasetRef, parameters: Optional[Mapping[str, Any]] = None) -
             # Check that the supplied parameters are suitable for the type read
             refStorageClass.validateParameters(parameters)
 
+            # We want to keep track of all the parameters that were not used
+            # by formatters.  We assume that if any of the component formatters
+            # use a parameter that we do not need to apply it again in the
+            # assembler.
             usedParams = set()
+
             components: Dict[str, Any] = {}
             for getInfo in allGetInfo:
                 # assemblerParams are parameters not understood by the
                 # associated formatter.
-                usedParams.update(set(getInfo.assemblerParams))
+                usedParams.update(set(getInfo.formatterParams))
 
                 component = getInfo.component
 
@@ -1015,7 +1024,7 @@ def get(self, ref: DatasetRef, parameters: Optional[Mapping[str, Any]] = None) -
             # Need to created a new info that specifies the read-only
             # component and associated storage class
             readInfo = DatastoreFileGetInformation(rwInfo.location, readFormatter,
-                                                   rwInfo.info, assemblerParams,
+                                                   rwInfo.info, assemblerParams, {},
                                                    refComponent, refStorageClass)
 
             return self._read_artifact_into_memory(readInfo, ref, isComponent=True)

python/lsst/daf/butler/datastores/posixDatastore.py

@@ -139,6 +139,9 @@ def _read_artifact_into_memory(self, getInfo: DatastoreFileGetInformation,
 
         formatter = getInfo.formatter
         try:
+            log.debug("Reading %s from location %s with formatter %s",
+                      f"component {getInfo.component}" if isComponent else "",
+                      location.uri, type(formatter).__name__)
             result = formatter.read(component=getInfo.component if isComponent else None)
         except Exception as e:
             raise ValueError(f"Failure from formatter '{formatter.name()}' for dataset {ref.id}"

tests/test_composites.py

@@ -43,10 +43,8 @@ def testConfig(self):
         self.assertIn("default", c)
         # Check merging has worked
         rootKey = "disassembled"
-        self.assertIn(f".{rootKey}.calexp", c)
         self.assertIn(f".{rootKey}.dummyTrue", c)
         self.assertIn(f".{rootKey}.StructuredComposite", c)
-        self.assertIn(f".{rootKey}.ExposureF", c)
 
         # Check that all entries are booleans (this is meant to be enforced
         # internally)