Merge branch 'tickets/DM-40210'

lsst · Aug 3, 2023 · d3b853d · d3b853d
2 parents 8639091 + 94300c6
commit d3b853d
Show file tree

Hide file tree

Showing 8 changed files with 41 additions and 74 deletions.
diff --git a/pipelines/DarkEnergyCamera/ApVerify.yaml b/pipelines/DarkEnergyCamera/ApVerify.yaml
@@ -15,15 +15,6 @@ tasks:
     class: lsst.ap.association.DiaPipelineTask
     config:
       doPackageAlerts: True
-  transformDiaSrcCat:
-    class: lsst.ap.association.TransformDiaSourceCatalogTask
-    config:
-      doIncludeReliability: True
-  rbClassify:
-    class: lsst.meas.transiNet.RBTransiNetTask
-    config:
-      modelPackageName: 'rbResnet50-DC2'
-      connections.coaddName: parameters.coaddName
 contracts:
   # Must re-declare contracts that cross apPipe and metrics boundary, as
   # these were removed on import.

diff --git a/pipelines/DarkEnergyCamera/ApVerifyWithFakes.yaml b/pipelines/DarkEnergyCamera/ApVerifyWithFakes.yaml
@@ -12,22 +12,5 @@ imports:
       - processCcd
   - location: $AP_PIPE_DIR/pipelines/DarkEnergyCamera/ProcessCcd.yaml
   # Can't use $AP_PIPE_DIR/pipelines/DarkEnergyCamera/ApPipeWithFakes.yaml here
-  # because the changes made by that file and ../ApVerifyWithFakes.yaml are
-  # hard to separate.
-tasks:
-  processVisitFakes:
-    class: lsst.pipe.tasks.processCcdWithFakes.ProcessCcdWithVariableFakesTask
-    config:
-      # Mirror calibrate config from DarkEnergyCamera/ProcessCcd.yaml
-      calibrate.photoCal.match.referenceSelection.magLimit.fluxField: i_flux
-      calibrate.photoCal.match.referenceSelection.magLimit.maximum: 22.0
-  transformDiaSrcCatWithFakes:
-    class: lsst.ap.association.TransformDiaSourceCatalogTask
-    config:
-      doIncludeReliability: True
-  rbClassify:
-    class: lsst.meas.transiNet.RBTransiNetTask
-    config:
-      modelPackageName: 'rbResnet50-DC2'
-      connections.coaddName: parameters.coaddName
-      connections.fakesType: parameters.fakesType
+  # because the changes made by that file and _ingredients/ApVerifyWithFakes.yaml
+  # are hard to separate.
diff --git a/pipelines/HyperSuprimeCam/ApVerify.yaml b/pipelines/HyperSuprimeCam/ApVerify.yaml
@@ -15,15 +15,6 @@ tasks:
     class: lsst.ap.association.DiaPipelineTask
     config:
       doPackageAlerts: True
-  transformDiaSrcCat:
-    class: lsst.ap.association.TransformDiaSourceCatalogTask
-    config:
-      doIncludeReliability: True
-  rbClassify:
-    class: lsst.meas.transiNet.RBTransiNetTask
-    config:
-      modelPackageName: 'rbResnet50-DC2'
-      connections.coaddName: parameters.coaddName
 contracts:
   # Must re-declare contracts that cross apPipe and metrics boundary, as
   # these were removed on import.

diff --git a/pipelines/HyperSuprimeCam/ApVerifyWithFakes.yaml b/pipelines/HyperSuprimeCam/ApVerifyWithFakes.yaml
@@ -9,17 +9,5 @@ imports:
       - processCcd
   - location: $AP_PIPE_DIR/pipelines/HyperSuprimeCam/ProcessCcd.yaml
   # Can't use $AP_PIPE_DIR/pipelines/HyperSuprimeCam/ApPipeWithFakes.yaml here
-  # because the changes made by that file and ../ApVerifyWithFakes.yaml are
-  # hard to separate.
-
-tasks:
-  transformDiaSrcCatWithFakes:
-    class: lsst.ap.association.TransformDiaSourceCatalogTask
-    config:
-      doIncludeReliability: True
-  rbClassify:
-    class: lsst.meas.transiNet.RBTransiNetTask
-    config:
-      modelPackageName: 'rbResnet50-DC2'
-      connections.coaddName: parameters.coaddName
-      connections.fakesType: parameters.fakesType
+  # because the changes made by that file and _ingredients/ApVerifyWithFakes.yaml
+  # are hard to separate.
diff --git a/pipelines/LSSTCam-imSim/ApVerify.yaml b/pipelines/LSSTCam-imSim/ApVerify.yaml
@@ -15,15 +15,6 @@ tasks:
     class: lsst.ap.association.DiaPipelineTask
     config:
       doPackageAlerts: True
-  transformDiaSrcCat:
-    class: lsst.ap.association.TransformDiaSourceCatalogTask
-    config:
-      doIncludeReliability: True
-  rbClassify:
-    class: lsst.meas.transiNet.RBTransiNetTask
-    config:
-      modelPackageName: 'rbResnet50-DC2'
-      connections.coaddName: parameters.coaddName
 contracts:
   # Must re-declare contracts that cross apPipe and metrics boundary, as
   # these were removed on import.

diff --git a/pipelines/LSSTCam-imSim/ApVerifyWithFakes.yaml b/pipelines/LSSTCam-imSim/ApVerifyWithFakes.yaml
@@ -9,17 +9,5 @@ imports:
       - processCcd
   - location: $AP_PIPE_DIR/pipelines/LsstCamImSim/ProcessCcd.yaml
   # Can't use $AP_PIPE_DIR/pipelines/LsstCamImSim/ApPipeWithFakes.yaml here
-  # because the changes made by that file and ../ApVerifyWithFakes.yaml are
-  # hard to separate.
-
-tasks:
-  transformDiaSrcCatWithFakes:
-    class: lsst.ap.association.TransformDiaSourceCatalogTask
-    config:
-      doIncludeReliability: True
-  rbClassify:
-    class: lsst.meas.transiNet.RBTransiNetTask
-    config:
-      modelPackageName: 'rbResnet50-DC2'
-      connections.coaddName: parameters.coaddName
-      connections.fakesType: parameters.fakesType
+  # because the changes made by that file and _ingredients/ApVerifyWithFakes.yaml
+  # are hard to separate.
diff --git a/pipelines/_ingredients/ApVerify.yaml b/pipelines/_ingredients/ApVerify.yaml
@@ -11,7 +11,24 @@ tasks:
   diaPipe:
     class: lsst.ap.association.DiaPipelineTask
     config:
+      # TODO: needed for "providing bulk sample alerts to brokers"; remove once
+      # we have an alternative.
       doPackageAlerts: True
+  # TODO: rbClassify and doIncludeReliability logically belong in ap_pipe, but
+  # having ap_pipe require manual setup of rbClassifier_data would cause too
+  # many problems. Move to ApPipe.yaml once either DM-38454 is implemented (and
+  # weights added to standard repos) or rbClassifier_data is fully integrated
+  # with the stack.
+  rbClassify:
+    class: lsst.meas.transiNet.RBTransiNetTask
+    config:
+      modelPackageStorageMode: neighbor  # Mode of rbResnet50-DC2
+      modelPackageName: rbResnet50-DC2   # Useful for non-DC2 data as well
+      connections.coaddName: parameters.coaddName
+  transformDiaSrcCat:
+    class: lsst.ap.association.TransformDiaSourceCatalogTask
+    config:
+      doIncludeReliability: True  # Output from rbClassify
 contracts:
   # Metric inputs must match pipeline outputs
   # Use of ConnectionsClass for templated fields is a workaround for DM-30210

diff --git a/pipelines/_ingredients/ApVerifyWithFakes.yaml b/pipelines/_ingredients/ApVerifyWithFakes.yaml
@@ -27,7 +27,25 @@ tasks:
   diaPipe:
     class: lsst.ap.association.DiaPipelineTask
     config:
+      # TODO: needed for "providing bulk sample alerts to brokers"; remove once
+      # we have an alternative.
       doPackageAlerts: True
+  # TODO: rbClassify and doIncludeReliability logically belong in ap_pipe, but
+  # having ap_pipe require manual setup of rbClassifier_data would cause too
+  # many problems. Move to ApPipeWithFakes.yaml once either DM-38454 is
+  # implemented (and weights added to standard repos) or rbClassifier_data is
+  # fully integrated with the stack.
+  rbClassifyWithFakes:
+    class: lsst.meas.transiNet.RBTransiNetTask
+    config:
+      modelPackageStorageMode: neighbor  # Mode of rbResnet50-DC2
+      modelPackageName: rbResnet50-DC2   # Useful for non-DC2 data as well
+      connections.coaddName: parameters.coaddName
+      connections.fakesType: parameters.fakesType
+  transformDiaSrcCatWithFakes:
+    class: lsst.ap.association.TransformDiaSourceCatalogTask
+    config:
+      doIncludeReliability: True  # Output from rbClassify
   timing_transformDiaSrcCat:
     class: lsst.verify.tasks.commonMetrics.TimingMetricTask
     config: