Merge branch 'tickets/DM-42217'

doc/lsst.ap.verify/datasets-creation.rst

@@ -31,7 +31,7 @@ Organizing the data
 
 * The :file:`raw` directory contains uningested science data.
   The directory may have any internal structure.
-* The :file:`preloaded` directory contains a :ref:`Gen 3 LSST Butler repository<lsst.daf.butler-using>` with calibration data, coadded difference imaging templates, refcats, and any other files needed for processing science data.
+* The :file:`preloaded` directory contains a :ref:`Gen 3 LSST Butler repository<lsst.daf.butler-using>` with calibration data, coadded difference imaging templates, refcats, pretrained models, and any other files needed for processing science data.
   The repository must have an ``<instrument>/defaults`` collection containing all of the above.
   It must not contain science data, which belongs only in :file:`raw`.
 * The :file:`config/export.yaml` file is a `relative-path export <lsst.daf.butler.Butler.export>` of the repository at :file:`preloaded`, used to set up a separate repository for running ``ap_verify``.

doc/lsst.ap.verify/running.rst

@@ -30,16 +30,6 @@ Using the `Cosmos PDR2`_ CI dataset as an example, first setup the dataset, if i
 
 You will need to setup the dataset once each session.
 
-Next, clone the `model packages`_ for the Real-Bogus (RB) Classifier and run the setup as follows.
-
-.. _model packages: https://github.com/lsst-dm/rbClassifier_data/
-
-.. prompt:: bash
-
-   setup [-r] rbClassifier_data
-
-You will need to setup the rbClassifier_data as well once each session.
-
 You can then run :command:`ap_verify.py` as follows.
 
 .. prompt:: bash

pipelines/_ingredients/ApVerify.yaml

@@ -17,21 +17,6 @@ tasks:
       # TODO: needed for "providing bulk sample alerts to brokers"; remove once
       # we have an alternative.
       doPackageAlerts: True
-  # TODO: rbClassify and doIncludeReliability logically belong in ap_pipe, but
-  # having ap_pipe require manual setup of rbClassifier_data would cause too
-  # many problems. Move to ApPipe.yaml once either DM-38454 is implemented (and
-  # weights added to standard repos) or rbClassifier_data is fully integrated
-  # with the stack.
-  rbClassify:
-    class: lsst.meas.transiNet.RBTransiNetTask
-    config:
-      modelPackageStorageMode: neighbor  # Mode of rbResnet50-DC2
-      modelPackageName: rbResnet50-DC2   # Useful for non-DC2 data as well
-      connections.coaddName: parameters.coaddName
-  transformDiaSrcCat:
-    class: lsst.ap.association.TransformDiaSourceCatalogTask
-    config:
-      doIncludeReliability: True  # Output from rbClassify
 contracts:
   # Metric inputs must match pipeline outputs
   # Use of ConnectionsClass for templated fields is a workaround for DM-30210

pipelines/_ingredients/ApVerifyCalibrateImage.yaml

@@ -18,19 +18,3 @@ tasks:
       # TODO: needed for "providing bulk sample alerts to brokers"; remove once
       # we have an alternative.
       doPackageAlerts: True
-  # TODO: rbClassify and doIncludeReliability logically belong in ap_pipe, but
-  # having ap_pipe require manual setup of rbClassifier_data would cause too
-  # many problems. Move to ApPipe.yaml once either DM-38454 is implemented (and
-  # weights added to standard repos) or rbClassifier_data is fully integrated
-  # with the stack.
-  rbClassify:
-    class: lsst.meas.transiNet.RBTransiNetTask
-    config:
-      modelPackageStorageMode: neighbor  # Mode of rbResnet50-DC2
-      modelPackageName: rbResnet50-DC2   # Useful for non-DC2 data as well
-      connections.coaddName: parameters.coaddName
-      connections.science: initial_pvi
-  transformDiaSrcCat:
-    class: lsst.ap.association.TransformDiaSourceCatalogTask
-    config:
-      doIncludeReliability: True  # Output from rbClassify

pipelines/_ingredients/ApVerifyWithFakes.yaml

@@ -33,22 +33,6 @@ tasks:
       # TODO: needed for "providing bulk sample alerts to brokers"; remove once
       # we have an alternative.
       doPackageAlerts: True
-  # TODO: rbClassify and doIncludeReliability logically belong in ap_pipe, but
-  # having ap_pipe require manual setup of rbClassifier_data would cause too
-  # many problems. Move to ApPipeWithFakes.yaml once either DM-38454 is
-  # implemented (and weights added to standard repos) or rbClassifier_data is
-  # fully integrated with the stack.
-  rbClassifyWithFakes:
-    class: lsst.meas.transiNet.RBTransiNetTask
-    config:
-      modelPackageStorageMode: neighbor  # Mode of rbResnet50-DC2
-      modelPackageName: rbResnet50-DC2   # Useful for non-DC2 data as well
-      connections.coaddName: parameters.coaddName
-      connections.fakesType: parameters.fakesType
-  transformDiaSrcCatWithFakes:
-    class: lsst.ap.association.TransformDiaSourceCatalogTask
-    config:
-      doIncludeReliability: True  # Output from rbClassify
   timing_transformDiaSrcCat:
     class: lsst.verify.tasks.commonMetrics.TimingMetricTask
     config:

python/lsst/ap/verify/testPipeline.py

@@ -39,6 +39,7 @@
 from lsst.ip.diffim import GetTemplateConfig, AlardLuptonSubtractConfig, DetectAndMeasureConfig
 from lsst.pipe.tasks.characterizeImage import CharacterizeImageConfig
 from lsst.pipe.tasks.calibrate import CalibrateConfig
+from lsst.meas.transiNet import RBTransiNetConfig
 from lsst.ap.association import TransformDiaSourceCatalogConfig, DiaPipelineConfig
 
 
@@ -433,6 +434,16 @@ def run(self, science, matchedTemplate, difference,
                       )
 
 
+class MockRBTransiNetTask(PipelineTask):
+    """A do-nothing substitute for RBTransiNetTask.
+    """
+    _DefaultName = "notRbTransiNet"
+    ConfigClass = RBTransiNetConfig
+
+    def run(self, template, science, difference, diaSources, pretrainedModel=None):
+        return Struct(classifications=afwTable.BaseCatalog(afwTable.Schema()))
+
+
 class MockTransformDiaSourceCatalogTask(PipelineTask):
     """A do-nothing substitute for TransformDiaSourceCatalogTask.
     """

tests/MockApPipe.yaml

@@ -35,6 +35,11 @@ tasks:
     config:
       connections.coaddName: parameters.coaddName
       doSkySources: True
+  rbClassify:
+    class: lsst.ap.verify.testPipeline.MockRBTransiNetTask
+    config:
+      modelPackageStorageMode: butler
+      connections.coaddName: parameters.coaddName
   transformDiaSrcCat:
     class: lsst.ap.verify.testPipeline.MockTransformDiaSourceCatalogTask
     config:
@@ -58,11 +63,18 @@ contracts:
       diaPipe.connections.ConnectionsClass(config=diaPipe).template.name
   - subtractImages.connections.ConnectionsClass(config=subtractImages).science.name ==
       diaPipe.connections.ConnectionsClass(config=diaPipe).exposure.name
+  - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
+      rbClassify.connections.ConnectionsClass(config=rbClassify).difference.name
+  - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).diaSources.name ==
+      rbClassify.connections.ConnectionsClass(config=rbClassify).diaSources.name
   - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
       transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diffIm.name
   - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).diaSources.name ==
       transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diaSourceCat.name
   - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
         diaPipe.connections.ConnectionsClass(config=diaPipe).diffIm.name
+  - (not transformDiaSrcCat.doIncludeReliability) or
+        (rbClassify.connections.ConnectionsClass(config=rbClassify).classifications.name ==
+            transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).reliability.name)
   - transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diaSourceTable.name ==
         diaPipe.connections.ConnectionsClass(config=diaPipe).diaSourceTable.name

tests/test_testPipeline.py

@@ -38,7 +38,7 @@
 from lsst.ap.verify.testPipeline import MockIsrTask, MockCharacterizeImageTask, \
     MockCalibrateTask, MockGetTemplateTask, \
     MockAlardLuptonSubtractTask, MockDetectAndMeasureTask, MockTransformDiaSourceCatalogTask, \
-    MockDiaPipelineTask
+    MockRBTransiNetTask, MockDiaPipelineTask
 
 
 class MockTaskTestSuite(unittest.TestCase):
@@ -82,6 +82,7 @@ def setUpClass(cls):
         butlerTests.addDataIdValue(cls.repo, "tract", TRACT)
         butlerTests.addDataIdValue(cls.repo, "patch", PATCH)
 
+        cls.emptyId = cls.repo.registry.expandDataId({})
         cls.exposureId = cls.repo.registry.expandDataId(
             {"instrument": INSTRUMENT, "exposure": VISIT, "detector": CCD})
         cls.visitId = cls.repo.registry.expandDataId(
@@ -124,6 +125,7 @@ def setUpClass(cls):
         butlerTests.addDatasetType(cls.repo, "apdb_marker", cls.visitId.dimensions, "Config")
         butlerTests.addDatasetType(cls.repo, "deepDiff_assocDiaSrc", cls.visitId.dimensions, "DataFrame")
         butlerTests.addDatasetType(cls.repo, "deepDiff_longTrailedSrc", cls.visitId.dimensions, "DataFrame")
+        butlerTests.addDatasetType(cls.repo, "deepRealBogusSources", cls.visitId.dimensions, "Catalog")
 
     def setUp(self):
         super().setUp()
@@ -249,6 +251,31 @@ def testMockDetectAndMeasureTask(self):
              })
         pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
 
+    def testMockRBTransiNetTask(self):
+        task = MockRBTransiNetTask()
+        pipelineTests.assertValidInitOutput(task)
+        result = task.run(template=afwImage.ExposureF(),
+                          science=afwImage.ExposureF(),
+                          difference=afwImage.ExposureF(),
+                          diaSources=afwTable.SourceCatalog(),
+                          )
+        pipelineTests.assertValidOutput(task, result)
+
+        self.butler.put(afwImage.ExposureF(), "calexp", self.visitId)
+        self.butler.put(afwImage.ExposureF(), "deepDiff_differenceExp", self.visitId)
+        self.butler.put(afwImage.ExposureF(), "deepDiff_templateExp", self.visitId)
+        self.butler.put(afwTable.SourceCatalog(), "deepDiff_diaSrc", self.visitId)
+        quantum = pipelineTests.makeQuantum(
+            task, self.butler, self.visitId,
+            {"template": self.visitId,
+             "science": self.visitId,
+             "difference": self.visitId,
+             "diaSources": self.visitId,
+             "pretrainedModel": self.emptyId,
+             "classifications": self.visitId,
+             })
+        pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
+
     def testMockTransformDiaSourceCatalogTask(self):
         task = MockTransformDiaSourceCatalogTask(initInputs=afwTable.SourceCatalog())
         pipelineTests.assertValidInitOutput(task)