Merge pull request #419 from lsst/tickets/DM-34919

DM-34919: Allow the generic butler tests to work with gen2 and gen3

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
       - id: end-of-file-fixer
       - id: trailing-whitespace
   - repo: https://github.com/psf/black
-    rev: 22.1.0
+    rev: 22.3.0
     hooks:
       - id: black
         # It is recommended to specify the latest version of Python

python/lsst/obs/base/butler_tests.py

@@ -24,6 +24,9 @@
 import inspect
 import unittest
 
+from lsst.daf.butler import Butler
+from lsst.daf.butler.registry import DataIdValueError
+
 __all__ = ["ButlerGetTests"]
 
 
@@ -130,7 +133,17 @@ def setUp_butler_get(
             raw_header_wcs=raw_header_wcs,
         )
 
+    def _require_gen2(self):
+        if isinstance(self.butler, Butler):
+            self.skipTest("This test requires daf_persistence Butler, not daf_butler Butler.")
+
+    def _is_gen3(self):
+        if isinstance(self.butler, Butler):
+            return True
+        return False
+
     def test_exposureId_bits(self):
+        self._require_gen2()
         bits = self.butler.get("ccdExposureId_bits")
         self.assertEqual(bits, self.butler_get_data.ccdExposureId_bits)
 
@@ -139,13 +152,17 @@ def _test_exposure(self, name):
             self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name))
         exp = self.butler.get(name, self.dataIds[name])
 
-        exp_md = self.butler.get(name + "_md", self.dataIds[name])
+        md_component = ".metadata" if self._is_gen3() else "_md"
+        exp_md = self.butler.get(name + md_component, self.dataIds[name])
         self.assertEqual(type(exp_md), type(exp.getMetadata()))
 
         self.assertEqual(exp.getDimensions(), self.butler_get_data.dimensions[name])
-        self.assertEqual(exp.getDetector().getId(), self.butler_get_data.detectorIds[name])
-        self.assertEqual(exp.getDetector().getName(), self.butler_get_data.detector_names[name])
-        self.assertEqual(exp.getDetector().getSerial(), self.butler_get_data.detector_serials[name])
+        detector = exp.detector
+        # Some calibration files are missing the detector.
+        if detector:
+            self.assertEqual(detector.getId(), self.butler_get_data.detectorIds[name])
+            self.assertEqual(detector.getName(), self.butler_get_data.detector_names[name])
+            self.assertEqual(detector.getSerial(), self.butler_get_data.detector_serials[name])
         # obs_test does not have physical filters, so include a fallback
         exposureFilter = exp.getFilterLabel()
         if exposureFilter:
@@ -156,8 +173,9 @@ def _test_exposure(self, name):
         else:
             filterName = "_unknown_"
         self.assertEqual(filterName, self.butler_get_data.filters[name])
-        exposureId = self.butler.get("ccdExposureId", dataId=self.dataIds[name])
-        self.assertEqual(exposureId, self.butler_get_data.exposureIds[name])
+        if not self._is_gen3():
+            exposureId = self.butler.get("ccdExposureId", dataId=self.dataIds[name])
+            self.assertEqual(exposureId, self.butler_get_data.exposureIds[name])
         self.assertEqual(exp.getInfo().getVisitInfo().getExposureTime(), self.butler_get_data.exptimes[name])
         return exp
 
@@ -186,22 +204,37 @@ def test_flat(self):
     def test_raw_header_wcs(self):
         """Test that `raw_header_wcs` returns the unmodified header of the raw
         image."""
-        if self.butler_get_data.raw_header_wcs is not None:
-            wcs = self.butler.get("raw_header_wcs", self.dataIds["raw"])
-            self.assertEqual(wcs, self.butler_get_data.raw_header_wcs)
-
-    @unittest.skip("Cannot test this, as there is a bug in the butler! DM-8097")
-    def test_raw_sub_bbox(self):
-        exp = self.butler.get("raw", self.dataIds["raw"], immediate=True)
-        bbox = exp.getBBox()
-        bbox.grow(-1)
-        sub = self.butler.get("raw_sub", self.dataIds["raw"], bbox=bbox, immediate=True)
-        self.assertEqual(sub.getImage().getBBox(), bbox)
-        self.assertImagesEqual(sub, exp.Factory(exp, bbox))
+        if self.butler_get_data.raw_header_wcs is None:
+            self.skipTest("Skipping raw header WCS test since no reference provided.")
+        # Gen3 will not understand this at the moment (DM-35031).
+        wcs = self.butler.get("raw_header_wcs", self.dataIds["raw"])
+        self.assertEqual(wcs, self.butler_get_data.raw_header_wcs)
 
     def test_subset_raw(self):
         for kwargs, expect in self.butler_get_data.raw_subsets:
-            subset = self.butler.subset("raw", **kwargs)
+            if self._is_gen3():
+                # If one of the keyword args looks like a dimension record
+                # subquery, pull it out into the WHERE clause.
+                where = []
+                bind = {}
+                for k, v in list(kwargs.items()):
+                    if "." in k:
+                        bindval = k.replace(".", "_")
+                        where.append(f"{k} = {bindval}")
+                        bind[bindval] = v
+                        del kwargs[k]
+                try:
+                    subset = set(
+                        self.butler.registry.queryDatasets(
+                            "raw", **kwargs, bind=bind, where=" AND ".join(where)
+                        )
+                    )
+                except DataIdValueError:
+                    # This means one of the dataId values does not exist.
+                    subset = {}
+            else:
+                subset = self.butler.subset("raw", **kwargs)
+
             self.assertEqual(len(subset), expect, msg="Failed for kwargs: {}".format(kwargs))
 
     def test_get_linearizer(self):
@@ -212,7 +245,11 @@ def test_get_linearizer(self):
         camera = self.butler.get("camera")
         for detectorId in self.butler_get_data.good_detectorIds:
             detector = camera[detectorId]
-            linearizer = self.butler.get("linearizer", dataId=dict(ccd=detectorId), immediate=True)
+            if self._is_gen3():
+                kwargs = {"detector": detectorId}
+            else:
+                kwargs = {"dataId": dict(ccd=detectorId), "immediate": True}
+            linearizer = self.butler.get("linearizer", **kwargs)
             self.assertEqual(linearizer.LinearityType, self.butler_get_data.linearizer_type[detectorId])
             linearizer.checkDetector(detector)
 
@@ -222,5 +259,9 @@ def test_get_linearizer_bad_detectorIds(self):
             self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name))
 
         for badccd in self.butler_get_data.bad_detectorIds:
+            if self._is_gen3():
+                kwargs = {"detector": badccd}
+            else:
+                kwargs = {"dataId": dict(ccd=badccd), "immediate": True}
             with self.assertRaises(RuntimeError):
-                self.butler.get("linearizer", dataId=dict(ccd=badccd), immediate=True)
+                self.butler.get("linearizer", **kwargs)

python/lsst/obs/base/camera_tests.py

@@ -25,6 +25,7 @@
 
 import lsst.geom
 from lsst.afw.cameraGeom import FIELD_ANGLE, FOCAL_PLANE
+from lsst.daf.butler import Butler
 
 __all__ = ["CameraTests"]
 
@@ -71,18 +72,31 @@ def setUp_camera(
             plate_scale=plate_scale,
         )
 
+    def _butler_args(self):
+        """Arguments to pass to butler get in addition to camera dataset type.
+
+        Returns
+        -------
+        kwargs : `dict`
+            The arguments to add.
+        """
+        kwargs = {}
+        if isinstance(self.butler, Butler):
+            kwargs = dict(instrument=self.camera_data.camera_name)
+        return kwargs
+
     def test_iterable(self):
         """Simplest camera test: can we get a Camera instance, and does
         iterating return Detectors?"""
-        camera = self.butler.get("camera", immediate=True)
+        camera = self.butler.get("camera", **self._butler_args())
         self.assertIsInstance(camera, lsst.afw.cameraGeom.Camera)
         for detector in camera:
             msg = "Failed for detector={}".format(detector)
             self.assertIsInstance(detector, lsst.afw.cameraGeom.Detector, msg=msg)
 
     def test_camera_butler(self):
         """Check that the butler returns the right type of camera."""
-        camera = self.butler.get("camera", immediate=True)
+        camera = self.butler.get("camera", **self._butler_args())
         self.assertEqual(camera.getName(), self.camera_data.camera_name)
         self.assertEqual(len(camera), self.camera_data.n_detectors)
         self.assertEqual(next(iter(camera)).getName(), self.camera_data.first_detector_name)
@@ -95,7 +109,7 @@ def test_plate_scale(self):
         """
         plate_scale = self.camera_data.plate_scale
         self.assertIsNotNone(plate_scale)
-        camera = self.butler.get("camera", immediate=True)
+        camera = self.butler.get("camera", **self._butler_args())
         focalPlaneToFieldAngle = camera.getTransformMap().getTransform(FOCAL_PLANE, FIELD_ANGLE)
         focalPlaneRadiusMm = 0.001  # an offset small enough to be in the linear regime
         for offsetAngleRad in (0.0, 0.65, 1.3):  # direction of offset; a few arbitrary angles

python/lsst/obs/base/mapper_tests.py

@@ -131,7 +131,12 @@ def setUp_mapper(
             test_config_metadata=test_config_metadata,
         )
 
+    def _check_mapper(self):
+        if self.mapper is None:
+            self.skipTest("No mapper defined. Likely a gen3 test environment.")
+
     def test_map_config_data(self):
+        self._check_mapper()
         if not self.mapper_data.test_config_metadata:
             self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name))
         dataId = self.dataIds["raw"]
@@ -148,6 +153,7 @@ def test_map_config_data(self):
             )
 
     def test_map_metadata_data(self):
+        self._check_mapper()
         if not self.mapper_data.test_config_metadata:
             self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name))
         dataId = self.dataIds["raw"]
@@ -162,23 +168,28 @@ def test_map_metadata_data(self):
             )
 
     def test_keys(self):
+        self._check_mapper()
         self.assertEqual(set(self.mapper.keys()), self.mapper_data.keys)
 
     def test_get_dataset_types(self):
+        self._check_mapper()
         if not self.mapper_data.test_config_metadata:
             self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name))
         someKeys = set(["raw", "processCcd_config", "processCcd_metadata"])
         self.assertTrue(set(self.mapper.getDatasetTypes()).issuperset(someKeys))
 
     def test_get_keys_raw(self):
+        self._check_mapper()
         for level, expect in self.mapper_data.raw_levels:
             result = self.mapper.getKeys("raw", level)
             self.assertEqual(set(result), expect, msg="Failed for level={}".format(level))
 
     def test_get_default_level(self):
+        self._check_mapper()
         self.assertEqual(self.mapper.getDefaultLevel(), self.mapper_data.default_level)
 
     def _test_map(self, butlerLocation, dataId):
+        self._check_mapper()
         self.assertEqual(butlerLocation.getPythonType(), self.mapper_data.map_python_type)
         self.assertEqual(butlerLocation.getCppType(), self.mapper_data.map_cpp_type)
         self.assertEqual(butlerLocation.getStorageName(), self.mapper_data.map_storage_name)
@@ -192,6 +203,7 @@ def _test_map(self, butlerLocation, dataId):
             )
 
     def test_map(self):
+        self._check_mapper()
         dataId = self.dataIds["raw"]
         location = self.mapper.map_raw(dataId)
         if not isinstance(location, lsst.daf.persistence.butlerLocation.ButlerComposite):
@@ -214,23 +226,27 @@ def test_query_metadata(self):
         Test expansion of incomplete information of the available data in this
         obs package's testdata repo.
         """
+        self._check_mapper()
         for query, expect in self.mapper_data.queryMetadata:
             # queryMetadata returns tuples of available items of the 2nd
             # parameter.
             result = self.mapper.queryMetadata("raw", self.mapper_data.query_format, query)
             self.assertEqual(sorted(result), sorted(expect), msg="Failed for query={}".format(query))
 
     def test_can_standardize(self):
+        self._check_mapper()
         self.assertTrue(self.mapper.canStandardize("raw"))
         self.assertFalse(self.mapper.canStandardize("camera"))
         if not self.mapper_data.test_config_metadata:
             self.assertFalse(self.mapper.canStandardize("processCcd_config"))
             self.assertFalse(self.mapper.canStandardize("processCcd_metadata"))
 
     def _test_validate(self, dataId):
+        self._check_mapper()
         self.assertEqual(self.mapper.validate(dataId), dataId)
 
     def test_validate(self):
+        self._check_mapper()
         self._test_validate({"visit": 1, "filter": "g"})
         self._test_validate({"visit": 2, "filter": "r"})
         self._test_validate({"visit": 3, "filter": "g", "tract": 4})