DM-43960: Spatial sample metrics task breaks fakes pipeline

pipelines/_ingredients/ApPipe.yaml

@@ -102,12 +102,18 @@ contracts:
   - contract: calibrateImage.connections.ConnectionsClass(config=calibrateImage).exposure.name ==
               rbClassify.connections.ConnectionsClass(config=rbClassify).science.name
     msg: "calibrateImage.exposure != rbClassify.science"
+  - contract: calibrateImage.connections.ConnectionsClass(config=calibrateImage).exposure.name ==
+              sampleSpatialMetrics.connections.ConnectionsClass(config=sampleSpatialMetrics).science.name
+    msg: "calibrateImage.exposure != sampleSpatialMetrics.science"
   - contract: calibrateImage.connections.ConnectionsClass(config=calibrateImage).exposure.name + ".summaryStats" ==
               initialPviCore.connections.ConnectionsClass(config=initialPviCore).data.name
     msg: "calibrateImage.exposure != initialPviCore.data"
   - contract: retrieveTemplate.connections.ConnectionsClass(config=retrieveTemplate).template.name ==
               subtractImages.connections.ConnectionsClass(config=subtractImages).template.name
     msg: "retrieveTemplate.template != subtractImages.template"
+  - contract: retrieveTemplate.connections.ConnectionsClass(config=retrieveTemplate).template.name ==
+              sampleSpatialMetrics.connections.ConnectionsClass(config=sampleSpatialMetrics).template.name
+    msg: "retrieveTemplate.template != sampleSpatialMetrics.template"
   - contract: subtractImages.connections.ConnectionsClass(config=subtractImages).difference.name ==
               detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).difference.name
     msg: "subtractImages.difference != detectAndMeasure.difference"
@@ -120,6 +126,9 @@ contracts:
   - contract: subtractImages.connections.ConnectionsClass(config=subtractImages).science.name ==
               diaPipe.connections.ConnectionsClass(config=diaPipe).exposure.name
     msg: "subtractImages.science != diaPipe.exposure"
+  - contract: subtractImages.connections.ConnectionsClass(config=subtractImages).matchedTemplate.name ==
+              sampleSpatialMetrics.connections.ConnectionsClass(config=sampleSpatialMetrics).matchedTemplate.name
+    msg: "subtractImages.matchedTemplate != sampleSpatialMetrics.matchedTemplate"
   - contract: detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).diaSources.name ==
               filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).diaSourceCat.name
     msg: "detectAndMeasure.diaSources != filterDiaSrcCat.diaSourceCat"
@@ -132,12 +141,18 @@ contracts:
   - contract: detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
               diaPipe.connections.ConnectionsClass(config=diaPipe).diffIm.name
     msg: "detectAndMeasure.subtractedMeasuredExposure != diaPipe.diffIm"
+  - contract: detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
+              sampleSpatialMetrics.connections.ConnectionsClass(config=sampleSpatialMetrics).difference.name
+    msg: "detectAndMeasure.subtractedMeasuredExposure != sampleSpatialMetrics.difference"
   - contract: filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).filteredDiaSourceCat.name ==
               rbClassify.connections.ConnectionsClass(config=rbClassify).diaSources.name
     msg: "filterDiaSrcCat.filteredDiaSourceCat != rbClassify.diaSources"
   - contract: filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).filteredDiaSourceCat.name ==
               transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diaSourceCat.name
     msg: "filterDiaSrcCat.filteredDiaSourceCat != transformDiaSrcCat.diaSourceCat"
+  - contract: filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).filteredDiaSourceCat.name ==
+              sampleSpatialMetrics.connections.ConnectionsClass(config=sampleSpatialMetrics).diaSources.name
+    msg: "filterDiaSrcCat.filteredDiaSourceCat != sampleSpatialMetrics.diaSources"
   - contract: (not transformDiaSrcCat.doIncludeReliability) or
               (rbClassify.connections.ConnectionsClass(config=rbClassify).classifications.name ==
                transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).reliability.name)

pipelines/_ingredients/ApPipeWithFakes.yaml

@@ -92,7 +92,6 @@ tasks:
   sampleSpatialMetrics:
     class: lsst.ip.diffim.SpatiallySampledMetricsTask
     config:
-      connections.science: initial_pvi
       connections.coaddName: parameters.coaddName
       connections.fakesType: parameters.fakesType
   fakesMatch:
@@ -133,6 +132,9 @@ contracts:
   # Inputs and outputs must match. For consistency, contracts are written in execution order:
   #     first task == second task, then sorted by (first, second)
   # Use of ConnectionsClass for templated fields is a workaround for DM-30210
+  - contract: processVisitFakes.connections.ConnectionsClass(config=processVisitFakes).outputExposure.name ==
+              sampleSpatialMetrics.connections.ConnectionsClass(config=sampleSpatialMetrics).science.name
+    msg: "processVisitFakes.outputExposure != sampleSpatialMetrics.science"
   - contract: createFakes.connections.ConnectionsClass(config=createFakes).fakeCat.name ==
               coaddFakes.connections.ConnectionsClass(config=coaddFakes).fakeCat.name
     msg: "createFakes.fakeCat != coaddFakes.fakeCat"
@@ -151,6 +153,9 @@ contracts:
   - contract: retrieveTemplate.connections.ConnectionsClass(config=retrieveTemplate).template.name ==
               subtractImages.connections.ConnectionsClass(config=subtractImages).template.name
     msg: "retrieveTemplate.template != subtractImages.template"
+  - contract: retrieveTemplate.connections.ConnectionsClass(config=retrieveTemplate).template.name ==
+              sampleSpatialMetrics.connections.ConnectionsClass(config=sampleSpatialMetrics).template.name
+    msg: "retrieveTemplate.template != sampleSpatialMetrics.template"
   - contract: subtractImages.connections.ConnectionsClass(config=subtractImages).difference.name ==
               detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).difference.name
     msg: "subtractImages.difference != detectAndMeasure.difference"
@@ -163,6 +168,9 @@ contracts:
   - contract: subtractImages.connections.ConnectionsClass(config=subtractImages).science.name ==
               diaPipe.connections.ConnectionsClass(config=diaPipe).exposure.name
     msg: "subtractImages.science != diaPipe.exposure"
+  - contract: subtractImages.connections.ConnectionsClass(config=subtractImages).matchedTemplate.name ==
+              sampleSpatialMetrics.connections.ConnectionsClass(config=sampleSpatialMetrics).matchedTemplate.name
+    msg: "subtractImages.matchedTemplate != sampleSpatialMetrics.matchedTemplate"
   - contract: detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).diaSources.name ==
               filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).diaSourceCat.name
     msg: "detectAndMeasure.diaSources != filterDiaSrcCat.diaSourceCat"
@@ -178,12 +186,18 @@ contracts:
   - contract: detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
               fakesMatch.connections.ConnectionsClass(config=fakesMatch).diffIm.name
     msg: "detectAndMeasure.subtractedMeasuredExposure != fakesMatch.diffIm"
+  - contract: detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
+              sampleSpatialMetrics.connections.ConnectionsClass(config=sampleSpatialMetrics).difference.name
+    msg: "detectAndMeasure.subtractedMeasuredExposure != sampleSpatialMetrics.difference"
   - contract: filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).filteredDiaSourceCat.name ==
               rbClassify.connections.ConnectionsClass(config=rbClassify).diaSources.name
     msg: "filterDiaSrcCat.filteredDiaSourceCat != rbClassify.diaSources"
   - contract: filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).filteredDiaSourceCat.name ==
               transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diaSourceCat.name
     msg: "filterDiaSrcCat.filteredDiaSourceCat != transformDiaSrcCat.diaSourceCat"
+  - contract: filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).filteredDiaSourceCat.name ==
+              sampleSpatialMetrics.connections.ConnectionsClass(config=sampleSpatialMetrics).diaSources.name
+    msg: "filterDiaSrcCat.filteredDiaSourceCat != sampleSpatialMetrics.diaSources"
   - contract: (not transformDiaSrcCat.doIncludeReliability) or
               (rbClassify.connections.ConnectionsClass(config=rbClassify).classifications.name ==
                transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).reliability.name)

tests/test_pipelines.py

@@ -21,12 +21,15 @@
 
 import glob
 import os.path
+import tempfile
 import unittest
 
 # need to import pyproj to prevent file handle leakage
 import pyproj  # noqa: F401
 
+import lsst.daf.butler.tests as butlerTests
 import lsst.pipe.base
+from lsst.pipe.base.tests.pipelineStepTester import PipelineStepTester  # Can't use fully-qualified name
 import lsst.utils
 import lsst.utils.tests
 
@@ -37,7 +40,7 @@ class PipelineDefintionsTestSuite(lsst.utils.tests.TestCase):
     def setUp(self):
         self.path = os.path.join(lsst.utils.getPackageDir("ap_pipe"), "pipelines")
 
-    def test_pipelines(self):
+    def test_graph_build(self):
         """Test that each pipeline definition file in `_ingredients/` can be
         used to build a graph.
         """
@@ -54,6 +57,41 @@ def test_pipelines(self):
                 # If this fails, it will produce a useful error message.
                 pipeline.to_graph()
 
+    def test_datasets(self):
+        files = glob.glob(os.path.join(self.path, "_ingredients/*.yaml"))
+        for file in files:
+            if "ApTemplate" in file:
+                # Our ApTemplate definition cannot be tested here because it
+                # depends on drp_tasks, which we cannot make a dependency here.
+                continue
+            with self.subTest(file):
+                tester = PipelineStepTester(
+                    filename=file,
+                    step_suffixes=[""],  # Test full pipeline
+                    initial_dataset_types=[("ps1_pv3_3pi_20170110", {"htm7"}, "SimpleCatalog", False),
+                                           ("gaia_dr2_20200414", {"htm7"}, "SimpleCatalog", False),
+                                           ("gaia_dr3_20230707", {"htm7"}, "SimpleCatalog", False),
+                                           ],
+                    expected_inputs={
+                        # ISR
+                        "raw", "camera", "crosstalk", "crosstalkSources", "bias", "dark", "flat", "ptc",
+                        "fringe", "straylightData", "bfKernel", "newBFKernel", "defects", "linearizer",
+                        "opticsTransmission", "filterTransmission", "atmosphereTransmission",
+                        "illumMaskedImage", "deferredChargeCalib",
+                        # Everything else
+                        "skyMap", "gaia_dr3_20230707", "gaia_dr2_20200414", "ps1_pv3_3pi_20170110",
+                        "goodSeeingCoadd", "pretrainedModelPackage",
+                    },
+                    # Pipeline outputs highly in flux, don't test
+                    expected_outputs=set(),
+                    pipeline_patches={"diaPipe:apdb.db_url": "sqlite://",
+                                      },
+                )
+                # Tester modifies Butler registry, so need a fresh repo every time
+                with tempfile.TemporaryDirectory() as tempRepo:
+                    butler = butlerTests.makeTestRepo(tempRepo)
+                    tester.run(butler, self)
+
 
 class MemoryTester(lsst.utils.tests.MemoryTestCase):
     pass