Add pipeline definitions for new CalibrateImage

This setup lets us run both the old (characterizeImage+calibrate) and new
(calibrateImage) pipelines as separate runs, so that we can directly compare
the new and old versions.

bps/clustering/clustering_ApPipe.yaml

@@ -19,6 +19,7 @@
 clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering
 cluster:
   singleFrame:
+    # TODO DM-40388: replace this with isr,calibrateImage
     pipetasks: isr,characterizeImage,calibrate
     dimensions: visit,detector
     equalDimensions: visit:exposure

bps/clustering/clustering_ApTemplate.yaml

@@ -12,6 +12,7 @@
 clusterAlgorithm: lsst.ctrl.bps.quantum_clustering_funcs.dimension_clustering
 cluster:
   singleFrame:
+    # TODO DM-40389: replace this with isr,calibrateImage
     pipetasks: isr,characterizeImage,calibrate
     dimensions: visit,detector
     equalDimensions: visit:exposure

pipelines/DarkEnergyCamera/ApPipeCalibrateImage.yaml

@@ -0,0 +1,28 @@
+# TODO DM-40388: This file will replace ApPipe.yaml, once the new pipeline is vetted.
+description: End to end AP pipeline specialized for DECam
+# (1) Run $AP_PIPE_DIR/pipelines/DarkEnergyCamera/RunIsrForCrosstalkSources.yaml
+# (2) Execute `make_apdb.py`, e.g.,
+#     make_apdb.py -c db_url="sqlite:////project/user/association.db"
+# (3) Run this pipeline, setting appropriate diaPipe configs
+#     (diaPipe configs should match the make_apdb.py configs)
+
+instrument: lsst.obs.decam.DarkEnergyCamera
+imports:
+  - location: $AP_PIPE_DIR/pipelines/DarkEnergyCamera/ProcessCcdCalibrateImage.yaml
+  - location: $AP_PIPE_DIR/pipelines/_ingredients/ApPipeCalibrateImage.yaml
+    exclude:  # These tasks come from DarkEnergyCamera/ProcessCcd.yaml instead
+      - processCcd
+subsets:
+# apPipe must be redefined from $AP_PIPE_DIR/pipelines/_ingredients/ApPipe.yaml
+# because its processCcd was excluded.
+  apPipe:
+    subset:
+      - isr
+      - calibrateImage
+      - retrieveTemplate
+      - subtractImages
+      - detectAndMeasure
+      - transformDiaSrcCat
+      - diaPipe
+    description: >
+      An alias of ApPipe to use in higher-level pipelines.

pipelines/DarkEnergyCamera/ProcessCcdCalibrateImage.yaml

@@ -0,0 +1,20 @@
+# TODO DM-40388: This file will replace ProcessCcd.yaml, once the new pipeline is vetted.
+description: A set of tasks to run when processing raw images, specialized for DECam
+# You must run $AP_PIPE_DIR/pipelines/DarkEnergyCamera/RunIsrForCrosstalkSources.yaml first!
+
+instrument: lsst.obs.decam.DarkEnergyCamera
+imports:
+  - location: $AP_PIPE_DIR/pipelines/DarkEnergyCamera/RunIsrWithCrosstalk.yaml
+  - location: $AP_PIPE_DIR/pipelines/_ingredients/ProcessCcdCalibrateImage.yaml
+    exclude:
+      - isr
+subsets:
+# processCcd must be redefined from
+# $AP_PIPE_DIR/pipelines/_ingredients/ProcessCcd.yaml because its isr was
+# excluded.
+  processCcd:
+    subset:
+      - isr
+      - calibrateImage
+    description: >
+      An alias of ProcessCcd to use in higher-level pipelines.

pipelines/HyperSuprimeCam/ApPipeCalibrateImage.yaml

@@ -0,0 +1,10 @@
+# TODO DM-40388: This file will replace ApPipe.yaml, once the new pipeline is vetted.
+description: End to end AP pipeline specialized for HSC
+# (1) Execute `make_apdb.py`, e.g.,
+#     make_apdb.py -c db_url="sqlite:////project/user/association.db"
+# (2) Run this pipeline, setting appropriate diaPipe configs
+#     (diaPipe configs should match the make_apdb.py configs)
+
+instrument: lsst.obs.subaru.HyperSuprimeCam
+imports:
+  - location: $AP_PIPE_DIR/pipelines/_ingredients/ApPipeCalibrateImage.yaml

pipelines/HyperSuprimeCam/ApTemplate.yaml

@@ -40,6 +40,8 @@ tasks:
       assembleStaticSkyModel.doSelectVisits: True
       connections.outputCoaddName: parameters.coaddName
 
+# TODO DM-40389: Remove this entire block once we are using calibrateImage in
+# the base _ingredients/ApTemplate.yaml.
 subsets:
 # The singleFrameAp subset is identical to the one in
 # $AP_PIPE_DIR/pipelines/_ingredients/ApTemplate.yaml, but needs to be defined

pipelines/HyperSuprimeCam/ProcessCcd.yaml

@@ -1,3 +1,4 @@
+# TODO DM-40388: Remove this file once the new pipeline is vetted.
 description: A set of tasks to run when processing raw images, specialized for HSC
 
 instrument: lsst.obs.subaru.HyperSuprimeCam

pipelines/LsstCamImSim/ApPipeCalibrateImage.yaml

@@ -0,0 +1,10 @@
+# TODO DM-40388: This file will replace ApPipe.yaml, once the new pipeline is vetted.
+description: End to end AP pipeline specialized for ImSim.
+# (1) Execute `make_apdb.py`, e.g.,
+#     make_apdb.py -c db_url="sqlite:////project/user/association.db"
+# (2) Run this pipeline, setting appropriate diaPipe configs
+#     (diaPipe configs should match the make_apdb.py configs)
+
+instrument: lsst.obs.lsst.LsstCamImSim
+imports:
+  - location: $AP_PIPE_DIR/pipelines/_ingredients/ApPipeCalibrateImage.yaml

pipelines/_ingredients/ApPipeCalibrateImage.yaml

@@ -0,0 +1,91 @@
+# TODO DM-40388: This file will replace ApPipe.yaml, once the new pipeline is vetted.
+description: End to end Alert Production pipeline.
+# Look in subdirectories of $AP_PIPE_DIR/pipelines to find customized pipelines
+# for each camera. Those pipelines import this general AP pipeline.
+#
+# NOTES
+# Remember to run make_apdb.py and use the same configs for diaPipe
+# A db_url is always required, e.g.,
+# -c diaPipe:apdb.db_url: 'sqlite:////project/user/association.db'
+# Option to specify connection_timeout for sqlite APDBs encountering lock errors, i.e.,
+# -c diaPipe:apdb.connection_timeout: 240
+
+# WARNING: camera-specific pipelines importing this pipeline may
+# blow away all the configs that are set in this file.
+# To update a pipeline config prior to DM-35504, you MUST put it in either,
+# e.g., $AP_PIPE_DIR/config/$CAMERA/someTask.py, or in a camera-specific,
+# pipeline, e.g., $AP_PIPE_DIR/pipelines/$CAMERA/ApPipeCalibrateImage.yaml.
+
+imports:
+  - location: $AP_PIPE_DIR/pipelines/_ingredients/ProcessCcdCalibrateImage.yaml
+parameters:
+  # Pipeline configurable to run on both goodSeeing and deep templates, depending on dataset.
+  coaddName: goodSeeing
+tasks:
+  retrieveTemplate:
+    class: lsst.ip.diffim.getTemplate.GetTemplateTask
+    config:
+      connections.bbox: initial_pvi.bbox
+      connections.wcs: initial_pvi.wcs
+      connections.coaddName: parameters.coaddName
+  subtractImages:
+    class: lsst.ip.diffim.subtractImages.AlardLuptonSubtractTask
+    config:
+      connections.science: initial_pvi
+      connections.sources: initial_stars_footprints_detector
+      connections.coaddName: parameters.coaddName
+      doApplyExternalCalibrations: False
+  detectAndMeasure:
+    class: lsst.ip.diffim.detectAndMeasure.DetectAndMeasureTask
+    config:
+      connections.science: initial_pvi
+      connections.coaddName: parameters.coaddName
+      doSkySources: True
+  transformDiaSrcCat:
+    class: lsst.ap.association.TransformDiaSourceCatalogTask
+    config:
+      doRemoveSkySources: True
+      connections.coaddName: parameters.coaddName
+  diaPipe:
+    class: lsst.ap.association.DiaPipelineTask
+    config:
+      connections.exposure: initial_pvi
+      connections.coaddName: parameters.coaddName
+subsets:
+  apPipe:
+    subset:
+      - isr
+      - calibrateImage
+      - retrieveTemplate
+      - subtractImages
+      - detectAndMeasure
+      - transformDiaSrcCat
+      - diaPipe
+    description: >
+      An alias of ApPipe to use in higher-level pipelines.
+contracts:
+  - detectAndMeasure.doSkySources == transformDiaSrcCat.doRemoveSkySources
+  # Inputs and outputs must match
+  # Use of ConnectionsClass for templated fields is a workaround for DM-30210
+  - calibrateImage.connections.ConnectionsClass(config=calibrateImage).output_exposure.name ==
+      subtractImages.connections.ConnectionsClass(config=subtractImages).science.name
+  - calibrateImage.connections.ConnectionsClass(config=calibrateImage).stars.name ==
+      subtractImages.connections.ConnectionsClass(config=subtractImages).sources.name
+  - retrieveTemplate.connections.ConnectionsClass(config=retrieveTemplate).template.name ==
+      subtractImages.connections.ConnectionsClass(config=subtractImages).template.name
+  - subtractImages.connections.ConnectionsClass(config=subtractImages).difference.name ==
+      detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).difference.name
+  - subtractImages.connections.ConnectionsClass(config=subtractImages).science.name ==
+      detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).science.name
+  - subtractImages.connections.ConnectionsClass(config=subtractImages).template.name ==
+      diaPipe.connections.ConnectionsClass(config=diaPipe).template.name
+  - subtractImages.connections.ConnectionsClass(config=subtractImages).science.name ==
+      diaPipe.connections.ConnectionsClass(config=diaPipe).exposure.name
+  - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
+      transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diffIm.name
+  - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).diaSources.name ==
+      transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diaSourceCat.name
+  - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
+        diaPipe.connections.ConnectionsClass(config=diaPipe).diffIm.name
+  - transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diaSourceTable.name ==
+        diaPipe.connections.ConnectionsClass(config=diaPipe).diaSourceTable.name

pipelines/_ingredients/ApTemplate.yaml

@@ -48,6 +48,7 @@ subsets:
   singleFrameAp:
     subset:
       - isr
+      # TODO DM-40389: replace these with calibrateImage
       - characterizeImage
       - calibrate
       - consolidateVisitSummary

pipelines/_ingredients/ProcessCcdCalibrateImage.yaml

@@ -0,0 +1,25 @@
+# TODO DM-40388: This file will replace ProcessCcd.yaml, once the new pipeline is vetted.
+description: ProcessCcd - A set of tasks to run when processing raw images.
+
+# WARNING: some camera-specific pipelines importing this pipeline presently
+# blow away all the configs that are set in this file.
+# To update a pipeline config prior to DM-35504, you MUST put it in either,
+# e.g., $AP_PIPE_DIR/config/$CAMERA/someTask.py, or in a camera-specific,
+# pipeline, e.g., $AP_PIPE_DIR/pipelines/$CAMERA/ProcessCcdCalibrateImage.yaml.
+
+tasks:
+  isr: lsst.ip.isr.IsrTask
+  calibrateImage:
+    class: lsst.pipe.tasks.calibrateImage.CalibrateImageTask
+    config:
+      # TODO DM-39796: Temporary, until these become the repair defaults.
+      python: >
+        config.psf_repair.cosmicray.nCrPixelMax = 1000000;
+        config.psf_repair.cosmicray.cond3_fac2 = 0.4;
+subsets:
+  processCcd:
+    subset:
+      - isr
+      - calibrateImage
+    description: >
+      An alias of ProcessCcd to use in higher-level pipelines.