DM-40388: Switch AP pipelines to use new CalibrateImageTask

pipelines/DECam/ApPipe.yaml

@@ -9,16 +9,15 @@ instrument: lsst.obs.decam.DarkEnergyCamera
 imports:
   - location: $AP_PIPE_DIR/pipelines/DECam/ProcessCcd.yaml
   - location: $AP_PIPE_DIR/pipelines/_ingredients/ApPipe.yaml
-    exclude:  # These tasks come from DECam/ProcessCcd.yaml instead
+    exclude:  # These tasks come from DECam/ProcessCcdCalibrateImage.yaml instead
       - processCcd
 subsets:
-# apPipe must be redefined from $AP_PIPE_DIR/pipelines/_ingredients/ApPipe.yaml
+# apPipe must be redefined from $AP_PIPE_DIR/pipelines/_ingredients/ApPipeCalibrateImage.yaml
 # because its processCcd was excluded.
   apPipe:
     subset:
       - isr
-      - characterizeImage
-      - calibrate
+      - calibrateImage
       - retrieveTemplate
       - subtractImages
       - detectAndMeasure

pipelines/DECam/ApPipeCalibrateImage.yaml → pipelines/DECam/ApPipeCalibrate.yaml

@@ -1,4 +1,3 @@
-# TODO DM-40388: This file will replace ApPipe.yaml, once the new pipeline is vetted.
 description: End to end AP pipeline specialized for DECam
 # (1) Run $AP_PIPE_DIR/pipelines/DECam/RunIsrForCrosstalkSources.yaml
 # (2) Execute `make_apdb.py`, e.g.,
@@ -8,17 +7,18 @@ description: End to end AP pipeline specialized for DECam
 
 instrument: lsst.obs.decam.DarkEnergyCamera
 imports:
-  - location: $AP_PIPE_DIR/pipelines/DECam/ProcessCcdCalibrateImage.yaml
-  - location: $AP_PIPE_DIR/pipelines/_ingredients/ApPipeCalibrateImage.yaml
-    exclude:  # These tasks come from DECam/ProcessCcdCalibrateImage.yaml instead
+  - location: $AP_PIPE_DIR/pipelines/DECam/ProcessCcdCalibrate.yaml
+  - location: $AP_PIPE_DIR/pipelines/_ingredients/ApPipeCalibrate.yaml
+    exclude:  # These tasks come from DECam/ProcessCcd.yaml instead
       - processCcd
 subsets:
-# apPipe must be redefined from $AP_PIPE_DIR/pipelines/_ingredients/ApPipeCalibrateImage.yaml
+# apPipe must be redefined from $AP_PIPE_DIR/pipelines/_ingredients/ApPipe.yaml
 # because its processCcd was excluded.
   apPipe:
     subset:
       - isr
-      - calibrateImage
+      - characterizeImage
+      - calibrate
       - retrieveTemplate
       - subtractImages
       - detectAndMeasure

pipelines/DECam/ProcessCcd.yaml

@@ -7,31 +7,13 @@ imports:
   - location: $AP_PIPE_DIR/pipelines/_ingredients/ProcessCcd.yaml
     exclude:
       - isr
-tasks:
-  characterizeImage:
-    class: lsst.pipe.tasks.characterizeImage.CharacterizeImageTask
-    config:
-      # HACK: workaround for DM-31047, remove once we're no longer loading
-      # DRP-specific obs configs.
-      file: $AP_PIPE_DIR/config/DECam/characterizeImage.py
-      # Config file wipes out all pre-existing configs, so copy base pipeline
-      # config on top.
-  calibrate:
-    class: lsst.pipe.tasks.calibrate.CalibrateTask
-    config:
-      # HACK: workaround for DM-31047, remove once we're no longer loading
-      # DRP-specific obs configs.
-      file: $AP_PIPE_DIR/config/DECam/calibrate.py
-      # Do not integrate file and pipeline configs until DM-31047 fixed, to
-      # make it easier to check for changes on the obs side.
 subsets:
 # processCcd must be redefined from
 # $AP_PIPE_DIR/pipelines/_ingredients/ProcessCcd.yaml because its isr was
 # excluded.
   processCcd:
     subset:
       - isr
-      - characterizeImage
-      - calibrate
+      - calibrateImage
     description: >
       An alias of ProcessCcd to use in higher-level pipelines.

pipelines/DECam/ProcessCcdCalibrate.yaml

@@ -0,0 +1,37 @@
+description: A set of tasks to run when processing raw images, specialized for DECam
+# You must run $AP_PIPE_DIR/pipelines/DECam/RunIsrForCrosstalkSources.yaml first!
+
+instrument: lsst.obs.decam.DarkEnergyCamera
+imports:
+  - location: $AP_PIPE_DIR/pipelines/DECam/RunIsrWithCrosstalk.yaml
+  - location: $AP_PIPE_DIR/pipelines/_ingredients/ProcessCcdCalibrate.yaml
+    exclude:
+      - isr
+tasks:
+  characterizeImage:
+    class: lsst.pipe.tasks.characterizeImage.CharacterizeImageTask
+    config:
+      # HACK: workaround for DM-31047, remove once we're no longer loading
+      # DRP-specific obs configs.
+      file: $AP_PIPE_DIR/config/DECam/characterizeImage.py
+      # Config file wipes out all pre-existing configs, so copy base pipeline
+      # config on top.
+  calibrate:
+    class: lsst.pipe.tasks.calibrate.CalibrateTask
+    config:
+      # HACK: workaround for DM-31047, remove once we're no longer loading
+      # DRP-specific obs configs.
+      file: $AP_PIPE_DIR/config/DECam/calibrate.py
+      # Do not integrate file and pipeline configs until DM-31047 fixed, to
+      # make it easier to check for changes on the obs side.
+subsets:
+# processCcd must be redefined from
+# $AP_PIPE_DIR/pipelines/_ingredients/ProcessCcd.yaml because its isr was
+# excluded.
+  processCcd:
+    subset:
+      - isr
+      - characterizeImage
+      - calibrate
+    description: >
+      An alias of ProcessCcd to use in higher-level pipelines.

pipelines/HSC/ApPipe.yaml

@@ -6,23 +6,4 @@ description: End to end AP pipeline specialized for HSC
 
 instrument: lsst.obs.subaru.HyperSuprimeCam
 imports:
-  - location: $AP_PIPE_DIR/pipelines/HSC/ProcessCcd.yaml
   - location: $AP_PIPE_DIR/pipelines/_ingredients/ApPipe.yaml
-    exclude:  # These tasks come from HSC/ProcessCcd.yaml instead
-      - processCcd
-subsets:
-# apPipe must be redefined from $AP_PIPE_DIR/pipelines/_ingredients/ApPipe.yaml
-# because its processCcd was excluded.
-  apPipe:
-    subset:
-      - isr
-      - characterizeImage
-      - calibrate
-      - retrieveTemplate
-      - subtractImages
-      - detectAndMeasure
-      - rbClassify
-      - transformDiaSrcCat
-      - diaPipe
-    description: >
-      An alias of ApPipe to use in higher-level pipelines.

pipelines/HSC/ApPipeCalibrate.yaml

@@ -0,0 +1,28 @@
+description: End to end AP pipeline specialized for HSC
+# (1) Execute `make_apdb.py`, e.g.,
+#     make_apdb.py -c db_url="sqlite:////project/user/association.db"
+# (2) Run this pipeline, setting appropriate diaPipe configs
+#     (diaPipe configs should match the make_apdb.py configs)
+
+instrument: lsst.obs.subaru.HyperSuprimeCam
+imports:
+  - location: $AP_PIPE_DIR/pipelines/HSC/ProcessCcdCalibrate.yaml
+  - location: $AP_PIPE_DIR/pipelines/_ingredients/ApPipeCalibrate.yaml
+    exclude:  # These tasks come from HSC/ProcessCcd.yaml instead
+      - processCcd
+subsets:
+# apPipe must be redefined from $AP_PIPE_DIR/pipelines/_ingredients/ApPipe.yaml
+# because its processCcd was excluded.
+  apPipe:
+    subset:
+      - isr
+      - characterizeImage
+      - calibrate
+      - retrieveTemplate
+      - subtractImages
+      - detectAndMeasure
+      - rbClassify
+      - transformDiaSrcCat
+      - diaPipe
+    description: >
+      An alias of ApPipe to use in higher-level pipelines.

pipelines/LSSTCam-imSim/ApPipe.yaml

@@ -1,28 +1,9 @@
-description: End to end AP pipeline specialized for ImSim
+description: End to end AP pipeline specialized for ImSim.
 # (1) Execute `make_apdb.py`, e.g.,
 #     make_apdb.py -c db_url="sqlite:////project/user/association.db"
 # (2) Run this pipeline, setting appropriate diaPipe configs
 #     (diaPipe configs should match the make_apdb.py configs)
 
 instrument: lsst.obs.lsst.LsstCamImSim
 imports:
-  - location: $AP_PIPE_DIR/pipelines/LSSTCam-imSim/ProcessCcd.yaml
   - location: $AP_PIPE_DIR/pipelines/_ingredients/ApPipe.yaml
-    exclude:  # These tasks come from LSSTCam-imSim/ProcessCcd.yaml instead
-      - processCcd
-subsets:
-# apPipe must be redefined from $AP_PIPE_DIR/pipelines/_ingredients/ApPipe.yaml
-# because its processCcd was excluded.
-  apPipe:
-    subset:
-      - isr
-      - characterizeImage
-      - calibrate
-      - retrieveTemplate
-      - subtractImages
-      - detectAndMeasure
-      - rbClassify
-      - transformDiaSrcCat
-      - diaPipe
-    description: >
-      An alias of ApPipe to use in higher-level pipelines.

pipelines/LSSTCam-imSim/ApPipeCalibrate.yaml

@@ -0,0 +1,28 @@
+description: End to end AP pipeline specialized for ImSim
+# (1) Execute `make_apdb.py`, e.g.,
+#     make_apdb.py -c db_url="sqlite:////project/user/association.db"
+# (2) Run this pipeline, setting appropriate diaPipe configs
+#     (diaPipe configs should match the make_apdb.py configs)
+
+instrument: lsst.obs.lsst.LsstCamImSim
+imports:
+  - location: $AP_PIPE_DIR/pipelines/LSSTCam-imSim/ProcessCcdCalibrate.yaml
+  - location: $AP_PIPE_DIR/pipelines/_ingredients/ApPipeCalibrate.yaml
+    exclude:  # These tasks come from LSSTCam-imSim/ProcessCcd.yaml instead
+      - processCcd
+subsets:
+# apPipe must be redefined from $AP_PIPE_DIR/pipelines/_ingredients/ApPipe.yaml
+# because its processCcd was excluded.
+  apPipe:
+    subset:
+      - isr
+      - characterizeImage
+      - calibrate
+      - retrieveTemplate
+      - subtractImages
+      - detectAndMeasure
+      - rbClassify
+      - transformDiaSrcCat
+      - diaPipe
+    description: >
+      An alias of ApPipe to use in higher-level pipelines.

pipelines/LSSTCam-imSim/ProcessCcd.yaml → pipelines/LSSTCam-imSim/ProcessCcdCalibrate.yaml

@@ -1,7 +1,7 @@
 description: Raw image pipeline specialized for ImSim
 instrument: lsst.obs.lsst.LsstCamImSim
 imports:
-  - location: $AP_PIPE_DIR/pipelines/_ingredients/ProcessCcd.yaml
+  - location: $AP_PIPE_DIR/pipelines/_ingredients/ProcessCcdCalibrate.yaml
 
 tasks:
   characterizeImage:

pipelines/_ingredients/ApPipe.yaml

@@ -1,4 +1,4 @@
-description: End to end Alert Production pipeline
+description: End to end Alert Production pipeline.
 # Look in subdirectories of $AP_PIPE_DIR/pipelines to find customized pipelines
 # for each camera. Those pipelines import this general AP pipeline.
 #
@@ -13,26 +13,31 @@ description: End to end Alert Production pipeline
 # blow away all the configs that are set in this file.
 # To update a pipeline config prior to DM-35504, you MUST put it in either,
 # e.g., $AP_PIPE_DIR/config/$CAMERA/someTask.py, or in a camera-specific,
-# pipeline, e.g., $AP_PIPE_DIR/pipelines/$CAMERA/ApPipe.yaml.
+# pipeline, e.g., $AP_PIPE_DIR/pipelines/$CAMERA/ApPipeCalibrateImage.yaml.
 
 imports:
   - location: $AP_PIPE_DIR/pipelines/_ingredients/ProcessCcd.yaml
 parameters:
   # Pipeline configurable to run on both goodSeeing and deep templates, depending on dataset.
   coaddName: goodSeeing
 tasks:
-  retrieveTemplate:  # For multi-tract difference imaging
+  retrieveTemplate:
     class: lsst.ip.diffim.getTemplate.GetTemplateTask
     config:
+      connections.bbox: initial_pvi.bbox
+      connections.wcs: initial_pvi.wcs
       connections.coaddName: parameters.coaddName
   subtractImages:
     class: lsst.ip.diffim.subtractImages.AlardLuptonSubtractTask
     config:
+      connections.science: initial_pvi
+      connections.sources: initial_stars_footprints_detector
       connections.coaddName: parameters.coaddName
       doApplyExternalCalibrations: False
   detectAndMeasure:
     class: lsst.ip.diffim.detectAndMeasure.DetectAndMeasureTask
     config:
+      connections.science: initial_pvi
       connections.coaddName: parameters.coaddName
       doSkySources: True
   filterDiaSrcCat:
@@ -44,6 +49,7 @@ tasks:
     class: lsst.meas.transiNet.RBTransiNetTask
     config:
       modelPackageStorageMode: butler
+      connections.science: initial_pvi
       connections.coaddName: parameters.coaddName
   transformDiaSrcCat:
     class: lsst.ap.association.TransformDiaSourceCatalogTask
@@ -54,27 +60,36 @@ tasks:
   diaPipe:
     class: lsst.ap.association.DiaPipelineTask
     config:
+      connections.exposure: initial_pvi
       connections.coaddName: parameters.coaddName
 subsets:
   apPipe:
     subset:
       - isr
-      - characterizeImage
-      - calibrate
+      - calibrateImage
       - retrieveTemplate
       - subtractImages
       - detectAndMeasure
+      - filterDiaSrcCat
       - rbClassify
       - transformDiaSrcCat
       - diaPipe
-      - filterDiaSrcCat
     description: >
       An alias of ApPipe to use in higher-level pipelines.
 contracts:
   - detectAndMeasure.doSkySources == transformDiaSrcCat.doRemoveSkySources
   # Inputs and outputs must match. For consistency, contracts are written in execution order:
   #     first task == second task, then sorted by (first, second)
   # Use of ConnectionsClass for templated fields is a workaround for DM-30210
+  - contract: calibrateImage.connections.ConnectionsClass(config=calibrateImage).output_exposure.name ==
+              subtractImages.connections.ConnectionsClass(config=subtractImages).science.name
+    msg: "calibrateImage.output_exposure != subtractImages.science"
+  - contract: calibrateImage.connections.ConnectionsClass(config=calibrateImage).stars_footprints.name ==
+              subtractImages.connections.ConnectionsClass(config=subtractImages).sources.name
+    msg: "calibrateImage.footprints_stars != subtractImages.sources"
+  - contract: calibrateImage.connections.ConnectionsClass(config=calibrateImage).output_exposure.name ==
+              rbClassify.connections.ConnectionsClass(config=rbClassify).science.name
+    msg: "calibrateImage.output_exposure != rbClassify.science"
   - contract: retrieveTemplate.connections.ConnectionsClass(config=retrieveTemplate).template.name ==
               subtractImages.connections.ConnectionsClass(config=subtractImages).template.name
     msg: "retrieveTemplate.template != subtractImages.template"