Add rbClassify task to AP pipeline.

This task uses a pretrained model in the Butler to characterize diaSources. Aside from the model (a PrerequisiteInput), its inputs and outputs integrate naturally with detectAndMeasure and transformDiaSrcCat.
lsst · Feb 2, 2024 · 843dcb9 · 843dcb9
1 parent 3e98878
commit 843dcb9
Show file tree

Hide file tree

Showing 10 changed files with 56 additions and 18 deletions.
diff --git a/pipelines/DECam/ApPipe.yaml b/pipelines/DECam/ApPipe.yaml
@@ -22,6 +22,7 @@ subsets:
       - retrieveTemplate
       - subtractImages
       - detectAndMeasure
+      - rbClassify
       - transformDiaSrcCat
       - diaPipe
     description: >

diff --git a/pipelines/DECam/ApPipeCalibrateImage.yaml b/pipelines/DECam/ApPipeCalibrateImage.yaml
@@ -22,6 +22,7 @@ subsets:
       - retrieveTemplate
       - subtractImages
       - detectAndMeasure
+      - rbClassify
       - transformDiaSrcCat
       - diaPipe
     description: >

diff --git a/pipelines/DECam/ApPipeWithFakes.yaml b/pipelines/DECam/ApPipeWithFakes.yaml
@@ -13,12 +13,8 @@ imports:
     exclude:  # These tasks frome from DECam's ApPipe.yaml instead
       - processCcd
   - location: $AP_PIPE_DIR/pipelines/DECam/ApPipe.yaml
-    exclude:  # These tasks come from ApPipeWithFakes.yaml instead
-      - retrieveTemplate
-      - subtractImages
-      - detectAndMeasure
-      - diaPipe
-      - transformDiaSrcCat
+    include:  # All other tasks come from ApPipeWithFakes.yaml instead
+      - processCcd
 
 tasks:
   diaPipe:

diff --git a/pipelines/HSC/ApPipe.yaml b/pipelines/HSC/ApPipe.yaml
@@ -21,6 +21,7 @@ subsets:
       - retrieveTemplate
       - subtractImages
       - detectAndMeasure
+      - rbClassify
       - transformDiaSrcCat
       - diaPipe
     description: >

diff --git a/pipelines/HSC/ApPipeWithFakes.yaml b/pipelines/HSC/ApPipeWithFakes.yaml
@@ -12,12 +12,8 @@ imports:
     exclude:  # These tasks frome from HSC's ApPipe.yaml instead
       - processCcd
   - location: $AP_PIPE_DIR/pipelines/HSC/ApPipe.yaml
-    exclude:  # These tasks come from _ingredients/ApPipeWithFakes.yaml instead
-      - retrieveTemplate
-      - subtractImages
-      - detectAndMeasure
-      - diaPipe
-      - transformDiaSrcCat
+    include:  # All other tasks come from _ingredients/ApPipeWithFakes.yaml instead
+      - processCcd
 
 tasks:
   processVisitFakes:

diff --git a/pipelines/LSSTCam-imSim/ApPipe.yaml b/pipelines/LSSTCam-imSim/ApPipe.yaml
@@ -21,6 +21,7 @@ subsets:
       - retrieveTemplate
       - subtractImages
       - detectAndMeasure
+      - rbClassify
       - transformDiaSrcCat
       - diaPipe
     description: >

diff --git a/pipelines/LSSTCam-imSim/ApPipeWithFakes.yaml b/pipelines/LSSTCam-imSim/ApPipeWithFakes.yaml
@@ -11,12 +11,8 @@ imports:
     exclude:  # These tasks come from LsstCamImSim/ApPipe.yaml instead
       - processCcd
   - location: $AP_PIPE_DIR/pipelines/LSSTCam-imSim/ApPipe.yaml
-    exclude:  # These tasks come from the ApPipeWithFakes.yaml
-      - retrieveTemplate
-      - subtractImages
-      - detectAndMeasure
-      - diaPipe
-      - transformDiaSrcCat
+    include:  # All other tasks come from the ApPipeWithFakes.yaml
+      - processCcd
 
 tasks:
   diaPipe:

diff --git a/pipelines/_ingredients/ApPipe.yaml b/pipelines/_ingredients/ApPipe.yaml
@@ -35,11 +35,17 @@ tasks:
     config:
       connections.coaddName: parameters.coaddName
       doSkySources: True
+  rbClassify:
+    class: lsst.meas.transiNet.RBTransiNetTask
+    config:
+      modelPackageStorageMode: butler
+      connections.coaddName: parameters.coaddName
   transformDiaSrcCat:
     class: lsst.ap.association.TransformDiaSourceCatalogTask
     config:
       doRemoveSkySources: True
       connections.coaddName: parameters.coaddName
+      doIncludeReliability: True  # Output from rbClassify
   diaPipe:
     class: lsst.ap.association.DiaPipelineTask
     config:
@@ -53,6 +59,7 @@ subsets:
       - retrieveTemplate
       - subtractImages
       - detectAndMeasure
+      - rbClassify
       - transformDiaSrcCat
       - diaPipe
     description: >
@@ -71,11 +78,18 @@ contracts:
       diaPipe.connections.ConnectionsClass(config=diaPipe).template.name
   - subtractImages.connections.ConnectionsClass(config=subtractImages).science.name ==
       diaPipe.connections.ConnectionsClass(config=diaPipe).exposure.name
+  - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
+      rbClassify.connections.ConnectionsClass(config=rbClassify).difference.name
+  - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).diaSources.name ==
+      rbClassify.connections.ConnectionsClass(config=rbClassify).diaSources.name
   - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
       transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diffIm.name
   - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).diaSources.name ==
       transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diaSourceCat.name
   - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
         diaPipe.connections.ConnectionsClass(config=diaPipe).diffIm.name
+  - (not transformDiaSrcCat.doIncludeReliability) or
+        (rbClassify.connections.ConnectionsClass(config=rbClassify).classifications.name ==
+            transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).reliability.name)
   - transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diaSourceTable.name ==
         diaPipe.connections.ConnectionsClass(config=diaPipe).diaSourceTable.name
diff --git a/pipelines/_ingredients/ApPipeCalibrateImage.yaml b/pipelines/_ingredients/ApPipeCalibrateImage.yaml
@@ -41,11 +41,18 @@ tasks:
       connections.science: initial_pvi
       connections.coaddName: parameters.coaddName
       doSkySources: True
+  rbClassify:
+    class: lsst.meas.transiNet.RBTransiNetTask
+    config:
+      modelPackageStorageMode: butler
+      connections.science: initial_pvi
+      connections.coaddName: parameters.coaddName
   transformDiaSrcCat:
     class: lsst.ap.association.TransformDiaSourceCatalogTask
     config:
       doRemoveSkySources: True
       connections.coaddName: parameters.coaddName
+      doIncludeReliability: True  # Output from rbClassify
   diaPipe:
     class: lsst.ap.association.DiaPipelineTask
     config:
@@ -59,6 +66,7 @@ subsets:
       - retrieveTemplate
       - subtractImages
       - detectAndMeasure
+      - rbClassify
       - transformDiaSrcCat
       - diaPipe
     description: >
@@ -71,6 +79,8 @@ contracts:
       subtractImages.connections.ConnectionsClass(config=subtractImages).science.name
   - calibrateImage.connections.ConnectionsClass(config=calibrateImage).stars.name ==
       subtractImages.connections.ConnectionsClass(config=subtractImages).sources.name
+  - calibrateImage.connections.ConnectionsClass(config=calibrateImage).output_exposure.name ==
+      rbClassify.connections.ConnectionsClass(config=rbClassify).science.name
   - retrieveTemplate.connections.ConnectionsClass(config=retrieveTemplate).template.name ==
       subtractImages.connections.ConnectionsClass(config=subtractImages).template.name
   - subtractImages.connections.ConnectionsClass(config=subtractImages).difference.name ==
@@ -81,11 +91,18 @@ contracts:
       diaPipe.connections.ConnectionsClass(config=diaPipe).template.name
   - subtractImages.connections.ConnectionsClass(config=subtractImages).science.name ==
       diaPipe.connections.ConnectionsClass(config=diaPipe).exposure.name
+  - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
+      rbClassify.connections.ConnectionsClass(config=rbClassify).difference.name
+  - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).diaSources.name ==
+        rbClassify.connections.ConnectionsClass(config=rbClassify).diaSources.name
   - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
       transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diffIm.name
   - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).diaSources.name ==
       transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diaSourceCat.name
   - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
         diaPipe.connections.ConnectionsClass(config=diaPipe).diffIm.name
+  - (not transformDiaSrcCat.doIncludeReliability) or
+        (rbClassify.connections.ConnectionsClass(config=rbClassify).classifications.name ==
+            transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).reliability.name)
   - transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diaSourceTable.name ==
         diaPipe.connections.ConnectionsClass(config=diaPipe).diaSourceTable.name
diff --git a/pipelines/_ingredients/ApPipeWithFakes.yaml b/pipelines/_ingredients/ApPipeWithFakes.yaml
@@ -57,12 +57,19 @@ tasks:
       connections.coaddName: parameters.coaddName
       connections.fakesType: parameters.fakesType
       doSkySources: True
+  rbClassifyWithFakes:
+    class: lsst.meas.transiNet.RBTransiNetTask
+    config:
+      modelPackageStorageMode: butler
+      connections.coaddName: parameters.coaddName
+      connections.fakesType: parameters.fakesType
   transformDiaSrcCatWithFakes:
     class: lsst.ap.association.TransformDiaSourceCatalogTask
     config:
       connections.coaddName: parameters.coaddName
       connections.fakesType: parameters.fakesType
       doRemoveSkySources: True
+      doIncludeReliability: True  # Output from rbClassifyWithFakes
   diaPipe:
     class: lsst.ap.association.DiaPipelineTask
     config:
@@ -90,6 +97,7 @@ subsets:
       - retrieveTemplateWithFakes
       - subtractImagesWithFakes
       - detectAndMeasureWithFakes
+      - rbClassifyWithFakes
       - transformDiaSrcCatWithFakes
       - diaPipe
       - fakesMatch
@@ -116,6 +124,10 @@ contracts:
       diaPipe.connections.ConnectionsClass(config=diaPipe).template.name
   - subtractImagesWithFakes.connections.ConnectionsClass(config=subtractImagesWithFakes).science.name ==
       diaPipe.connections.ConnectionsClass(config=diaPipe).exposure.name
+  - detectAndMeasureWithFakes.connections.ConnectionsClass(config=detectAndMeasureWithFakes).subtractedMeasuredExposure.name ==
+      rbClassifyWithFakes.connections.ConnectionsClass(config=rbClassifyWithFakes).difference.name
+  - detectAndMeasureWithFakes.connections.ConnectionsClass(config=detectAndMeasureWithFakes).diaSources.name ==
+      rbClassifyWithFakes.connections.ConnectionsClass(config=rbClassifyWithFakes).diaSources.name
   - detectAndMeasureWithFakes.connections.ConnectionsClass(config=detectAndMeasureWithFakes).subtractedMeasuredExposure.name ==
       transformDiaSrcCatWithFakes.connections.ConnectionsClass(config=transformDiaSrcCatWithFakes).diffIm.name
   - detectAndMeasureWithFakes.connections.ConnectionsClass(config=detectAndMeasureWithFakes).diaSources.name ==
@@ -124,6 +136,9 @@ contracts:
       diaPipe.connections.ConnectionsClass(config=diaPipe).diffIm.name
   - detectAndMeasureWithFakes.connections.ConnectionsClass(config=detectAndMeasureWithFakes).subtractedMeasuredExposure.name ==
       fakesMatch.connections.ConnectionsClass(config=fakesMatch).diffIm.name
+  - (not transformDiaSrcCatWithFakes.doIncludeReliability) or
+        (rbClassifyWithFakes.connections.ConnectionsClass(config=rbClassifyWithFakes).classifications.name ==
+            transformDiaSrcCatWithFakes.connections.ConnectionsClass(config=transformDiaSrcCatWithFakes).reliability.name)
   - transformDiaSrcCatWithFakes.connections.ConnectionsClass(config=transformDiaSrcCatWithFakes).diaSourceTable.name ==
       diaPipe.connections.ConnectionsClass(config=diaPipe).diaSourceTable.name
   - processVisitFakes.connections.ConnectionsClass(config=processVisitFakes).outputExposure.name ==