Allow the generic butler tests to work with gen2 and gen3

The mapper tests now disable themselves if no mapper is
available.

python/lsst/obs/base/butler_tests.py

@@ -23,6 +23,8 @@
 import collections
 import inspect
 import unittest
+from lsst.daf.butler import Butler
+from lsst.daf.butler.registry import DataIdValueError
 
 __all__ = ["ButlerGetTests"]
 
@@ -130,7 +132,17 @@ def setUp_butler_get(
             raw_header_wcs=raw_header_wcs,
         )
 
+    def _require_gen2(self):
+        if isinstance(self.butler, Butler):
+            self.skipTest("This test requires daf_persistence Butler, not daf_butler Butler.")
+
+    def _is_gen3(self):
+        if isinstance(self.butler, Butler):
+            return True
+        return False
+
     def test_exposureId_bits(self):
+        self._require_gen2()
         bits = self.butler.get("ccdExposureId_bits")
         self.assertEqual(bits, self.butler_get_data.ccdExposureId_bits)
 
@@ -139,13 +151,17 @@ def _test_exposure(self, name):
             self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name))
         exp = self.butler.get(name, self.dataIds[name])
 
-        exp_md = self.butler.get(name + "_md", self.dataIds[name])
+        md_component = ".metadata" if self._is_gen3() else "_md"
+        exp_md = self.butler.get(name + md_component, self.dataIds[name])
         self.assertEqual(type(exp_md), type(exp.getMetadata()))
 
         self.assertEqual(exp.getDimensions(), self.butler_get_data.dimensions[name])
-        self.assertEqual(exp.getDetector().getId(), self.butler_get_data.detectorIds[name])
-        self.assertEqual(exp.getDetector().getName(), self.butler_get_data.detector_names[name])
-        self.assertEqual(exp.getDetector().getSerial(), self.butler_get_data.detector_serials[name])
+        detector = exp.getDetector()
+        if detector:
+            # Some calibration files are missing the detector.
+            self.assertEqual(detector.getId(), self.butler_get_data.detectorIds[name])
+            self.assertEqual(detector.getName(), self.butler_get_data.detector_names[name])
+            self.assertEqual(detector.getSerial(), self.butler_get_data.detector_serials[name])
         # obs_test does not have physical filters, so include a fallback
         exposureFilter = exp.getFilterLabel()
         if exposureFilter:
@@ -156,8 +172,9 @@ def _test_exposure(self, name):
         else:
             filterName = "_unknown_"
         self.assertEqual(filterName, self.butler_get_data.filters[name])
-        exposureId = self.butler.get("ccdExposureId", dataId=self.dataIds[name])
-        self.assertEqual(exposureId, self.butler_get_data.exposureIds[name])
+        if not self._is_gen3():
+            exposureId = self.butler.get("ccdExposureId", dataId=self.dataIds[name])
+            self.assertEqual(exposureId, self.butler_get_data.exposureIds[name])
         self.assertEqual(exp.getInfo().getVisitInfo().getExposureTime(), self.butler_get_data.exptimes[name])
         return exp
 
@@ -186,12 +203,16 @@ def test_flat(self):
     def test_raw_header_wcs(self):
         """Test that `raw_header_wcs` returns the unmodified header of the raw
         image."""
-        if self.butler_get_data.raw_header_wcs is not None:
-            wcs = self.butler.get("raw_header_wcs", self.dataIds["raw"])
-            self.assertEqual(wcs, self.butler_get_data.raw_header_wcs)
+        if self.butler_get_data.raw_header_wcs is None:
+            self.skipTest("Skipping raw header WCS test since no reference provided.")
+        # Gen3 will not understand this at the moment.
+        wcs = self.butler.get("raw_header_wcs", self.dataIds["raw"])
+        self.assertEqual(wcs, self.butler_get_data.raw_header_wcs)
 
-    @unittest.skip("Cannot test this, as there is a bug in the butler! DM-8097")
+    @unittest.skip("Cannot test this. Raw data can not reliably have bbox requested.")
     def test_raw_sub_bbox(self):
+        # Gen2 variant only. Gen3 will never support this.
+        self._require_gen2()
         exp = self.butler.get("raw", self.dataIds["raw"], immediate=True)
         bbox = exp.getBBox()
         bbox.grow(-1)
@@ -201,7 +222,26 @@ def test_raw_sub_bbox(self):
 
     def test_subset_raw(self):
         for kwargs, expect in self.butler_get_data.raw_subsets:
-            subset = self.butler.subset("raw", **kwargs)
+            if self._is_gen3():
+                # If one of the keyword args looks like a dimension record
+                # subquery, pull it out into the WHERE clause.
+                where = []
+                bind = {}
+                for k, v in list(kwargs.items()):
+                    if "." in k:
+                        bindval = k.replace(".", "_")
+                        where.append(f"{k} = {bindval}")
+                        bind[bindval] = v
+                        del kwargs[k]
+                try:
+                    subset = set(self.butler.registry.queryDatasets("raw", **kwargs, bind=bind,
+                                                                    where=" AND ".join(where)))
+                except DataIdValueError:
+                    # This means one of the dataId values does not exist.
+                    subset = {}
+            else:
+                subset = self.butler.subset("raw", **kwargs)
+
             self.assertEqual(len(subset), expect, msg="Failed for kwargs: {}".format(kwargs))
 
     def test_get_linearizer(self):
@@ -212,7 +252,11 @@ def test_get_linearizer(self):
         camera = self.butler.get("camera")
         for detectorId in self.butler_get_data.good_detectorIds:
             detector = camera[detectorId]
-            linearizer = self.butler.get("linearizer", dataId=dict(ccd=detectorId), immediate=True)
+            if self._is_gen3():
+                kwargs = {"detector": detectorId}
+            else:
+                kwargs = {"dataId": dict(ccd=detectorId), "immediate": True}
+            linearizer = self.butler.get("linearizer", **kwargs)
             self.assertEqual(linearizer.LinearityType, self.butler_get_data.linearizer_type[detectorId])
             linearizer.checkDetector(detector)
 
@@ -222,5 +266,9 @@ def test_get_linearizer_bad_detectorIds(self):
             self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name))
 
         for badccd in self.butler_get_data.bad_detectorIds:
+            if self._is_gen3():
+                kwargs = {"detector": badccd}
+            else:
+                kwargs = {"dataId": dict(ccd=badccd), "immediate": True}
             with self.assertRaises(RuntimeError):
-                self.butler.get("linearizer", dataId=dict(ccd=badccd), immediate=True)
+                self.butler.get("linearizer", **kwargs)

python/lsst/obs/base/camera_tests.py

@@ -25,6 +25,7 @@
 
 import lsst.geom
 from lsst.afw.cameraGeom import FIELD_ANGLE, FOCAL_PLANE
+from lsst.daf.butler import Butler
 
 __all__ = ["CameraTests"]
 
@@ -71,18 +72,26 @@ def setUp_camera(
             plate_scale=plate_scale,
         )
 
+    def _butler_args(self):
+        """Arguments to pass to butler get in addition to camera dataset type.
+        """
+        kwargs = {}
+        if isinstance(self.butler, Butler):
+            kwargs = dict(instrument=self.camera_data.camera_name)
+        return kwargs
+
     def test_iterable(self):
         """Simplest camera test: can we get a Camera instance, and does
         iterating return Detectors?"""
-        camera = self.butler.get("camera", immediate=True)
+        camera = self.butler.get("camera", **self._butler_args())
         self.assertIsInstance(camera, lsst.afw.cameraGeom.Camera)
         for detector in camera:
             msg = "Failed for detector={}".format(detector)
             self.assertIsInstance(detector, lsst.afw.cameraGeom.Detector, msg=msg)
 
     def test_camera_butler(self):
         """Check that the butler returns the right type of camera."""
-        camera = self.butler.get("camera", immediate=True)
+        camera = self.butler.get("camera", **self._butler_args())
         self.assertEqual(camera.getName(), self.camera_data.camera_name)
         self.assertEqual(len(camera), self.camera_data.n_detectors)
         self.assertEqual(next(iter(camera)).getName(), self.camera_data.first_detector_name)
@@ -95,7 +104,7 @@ def test_plate_scale(self):
         """
         plate_scale = self.camera_data.plate_scale
         self.assertIsNotNone(plate_scale)
-        camera = self.butler.get("camera", immediate=True)
+        camera = self.butler.get("camera", **self._butler_args())
         focalPlaneToFieldAngle = camera.getTransformMap().getTransform(FOCAL_PLANE, FIELD_ANGLE)
         focalPlaneRadiusMm = 0.001  # an offset small enough to be in the linear regime
         for offsetAngleRad in (0.0, 0.65, 1.3):  # direction of offset; a few arbitrary angles

python/lsst/obs/base/mapper_tests.py

@@ -131,7 +131,12 @@ def setUp_mapper(
             test_config_metadata=test_config_metadata,
         )
 
+    def _check_mapper(self):
+        if self.mapper is None:
+            self.skipTest("No mapper defined. Likely a gen3 test environment.")
+
     def test_map_config_data(self):
+        self._check_mapper()
         if not self.mapper_data.test_config_metadata:
             self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name))
         dataId = self.dataIds["raw"]
@@ -148,6 +153,7 @@ def test_map_config_data(self):
             )
 
     def test_map_metadata_data(self):
+        self._check_mapper()
         if not self.mapper_data.test_config_metadata:
             self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name))
         dataId = self.dataIds["raw"]
@@ -162,23 +168,28 @@ def test_map_metadata_data(self):
             )
 
     def test_keys(self):
+        self._check_mapper()
         self.assertEqual(set(self.mapper.keys()), self.mapper_data.keys)
 
     def test_get_dataset_types(self):
+        self._check_mapper()
         if not self.mapper_data.test_config_metadata:
             self.skipTest("Skipping %s as requested" % (inspect.currentframe().f_code.co_name))
         someKeys = set(["raw", "processCcd_config", "processCcd_metadata"])
         self.assertTrue(set(self.mapper.getDatasetTypes()).issuperset(someKeys))
 
     def test_get_keys_raw(self):
+        self._check_mapper()
         for level, expect in self.mapper_data.raw_levels:
             result = self.mapper.getKeys("raw", level)
             self.assertEqual(set(result), expect, msg="Failed for level={}".format(level))
 
     def test_get_default_level(self):
+        self._check_mapper()
         self.assertEqual(self.mapper.getDefaultLevel(), self.mapper_data.default_level)
 
     def _test_map(self, butlerLocation, dataId):
+        self._check_mapper()
         self.assertEqual(butlerLocation.getPythonType(), self.mapper_data.map_python_type)
         self.assertEqual(butlerLocation.getCppType(), self.mapper_data.map_cpp_type)
         self.assertEqual(butlerLocation.getStorageName(), self.mapper_data.map_storage_name)
@@ -192,6 +203,7 @@ def _test_map(self, butlerLocation, dataId):
             )
 
     def test_map(self):
+        self._check_mapper()
         dataId = self.dataIds["raw"]
         location = self.mapper.map_raw(dataId)
         if not isinstance(location, lsst.daf.persistence.butlerLocation.ButlerComposite):
@@ -214,23 +226,27 @@ def test_query_metadata(self):
         Test expansion of incomplete information of the available data in this
         obs package's testdata repo.
         """
+        self._check_mapper()
         for query, expect in self.mapper_data.queryMetadata:
             # queryMetadata returns tuples of available items of the 2nd
             # parameter.
             result = self.mapper.queryMetadata("raw", self.mapper_data.query_format, query)
             self.assertEqual(sorted(result), sorted(expect), msg="Failed for query={}".format(query))
 
     def test_can_standardize(self):
+        self._check_mapper()
         self.assertTrue(self.mapper.canStandardize("raw"))
         self.assertFalse(self.mapper.canStandardize("camera"))
         if not self.mapper_data.test_config_metadata:
             self.assertFalse(self.mapper.canStandardize("processCcd_config"))
             self.assertFalse(self.mapper.canStandardize("processCcd_metadata"))
 
     def _test_validate(self, dataId):
+        self._check_mapper()
         self.assertEqual(self.mapper.validate(dataId), dataId)
 
     def test_validate(self):
+        self._check_mapper()
         self._test_validate({"visit": 1, "filter": "g"})
         self._test_validate({"visit": 2, "filter": "r"})
         self._test_validate({"visit": 3, "filter": "g", "tract": 4})