DM-32694: Split AP pipeline into ApPipeWithFakes

pipelines/ApVerify.yaml

@@ -13,5 +13,6 @@ tasks:
       doPackageAlerts: True
 contracts:
   # Metric inputs must match pipeline outputs
-  - imageDifference.connections.coaddName == fracDiaSourcesToSciSources.connections.coaddName
-  - imageDifference.connections.fakesType == fracDiaSourcesToSciSources.connections.fakesType
+  # Use of ConnectionsClass for templated fields is a workaround for DM-30210
+  - imageDifference.connections.ConnectionsClass(config=imageDifference).diaSources.name ==
+      fracDiaSourcesToSciSources.connections.ConnectionsClass(config=fracDiaSourcesToSciSources).diaSources.name

pipelines/ApVerifyWithFakes.yaml

@@ -3,211 +3,50 @@
 
 description: Fully instrumented AP pipeline with fakes
 imports:
-  - location: $AP_VERIFY_DIR/pipelines/ApVerify.yaml
-    exclude:
-      - fracDiaSourcesToSciSources  # Selectively turn off this task's contracts
+  - location: $AP_PIPE_DIR/pipelines/ApPipeWithFakes.yaml
+  - location: $AP_VERIFY_DIR/pipelines/MetricsRuntime.yaml
+  - location: $AP_VERIFY_DIR/pipelines/MetricsMisc.yaml
   - location: $AP_VERIFY_DIR/pipelines/MetricsForFakes.yaml
-parameters:
-  fakesType: fakes_
-  # TODO: redundant connection definitions workaround for DM-30210
-  fakesInput: fakes_fakeSourceCat
-  fakesPvi: fakes_calexp
-  fakesTemplate: fakes_goodSeeingCoadd
-  fakesDcrTemplate: fakes_dcrCoadd
-  fakesDiaSrcCat: fakes_goodSeeingDiff_diaSrc
-  fakesDiaSrcSchema: fakes_goodSeeingDiff_diaSrc_schema
-  fakesDiaSrcParquet: fakes_goodSeeingDiff_diaSrcTable
-  fakesDiff: fakes_goodSeeingDiff_differenceExp
-  fakesDiffScore: fakes_goodSeeingDiff_scoreExp
-  fakesDiffWarp: fakes_goodSeeingDiff_warpedExp
-  fakesDiffMatch: fakes_goodSeeingDiff_matchedExp
-  fakesAssocSrc: fakes_goodSeeingDiff_assocDiaSrc
-  fakesMatchedSrc: fakes_goodSeeingDiff_matchDiaSrc
-  # TODO: end DM-30210 workaround
 tasks:
-  createFakes:
-    class: lsst.ap.pipe.createApFakes.CreateRandomApFakesTask
-    config:
-      magMin: 20
-      magMax: 26
-      fakeDensity: 2000
-      connections.fakesType: parameters.fakesType
-      # TODO: redundant connection definitions workaround for DM-30210
-      connections.fakeCat: parameters.fakesInput
-      # TODO: end DM-30210 workaround
-  coaddFakes:
-    class: lsst.pipe.tasks.insertFakes.InsertFakesTask
-    config:
-      doSubSelectSources: True
-      select_col: "isTemplateSource"
-      connections.coaddName: parameters.coaddName
-      connections.fakesType: parameters.fakesType
-      # TODO: redundant connection definitions workaround for DM-30210
-      connections.image: parameters.template
-      connections.fakeCat: parameters.fakesInput
-      connections.imageWithFakes: parameters.fakesTemplate
-      # TODO: end DM-30210 workaround
-  singleFrameWithFakes:
-    class: lsst.pipe.tasks.processCcdWithFakes.ProcessCcdWithVariableFakesTask
-    config:
-      insertFakes.doSubSelectSources: True
-      insertFakes.select_col: "isVisitSource"
-  retrieveTemplateWithFakes:
+  # Parallel to ApPipe's retrieveTemplateWithFakes, allows non-fakes image differencing.
+  retrieveTemplate:
     class: lsst.ip.diffim.getTemplate.GetTemplateTask
     config:
-      connections.coaddName: parameters.coaddName
-      connections.fakesType: parameters.fakesType
-
-  # Contracts require imageDifference, transformDiaSrc, and diaPipe to have matching configs.
-  # Since diaPipe must include fakes (see below), do non-fakes processing with non-standard names.
-  imageDifferenceNoFakes:
+        connections.coaddName: parameters.coaddName
+  # Parallel to ApPipe's imageDifferenceWithFakes, allows clean differencing metrics.
+  imageDifference:
     class: lsst.pipe.tasks.imageDifference.ImageDifferenceFromTemplateTask
     config:
-      doWriteWarpedExp: True       # Required for packaging alerts in diaPipe
+      doWriteWarpedExp: True       # Required for diaPipe, keep for timing/metrics purposes
       doSkySources: True
       coaddName: parameters.coaddName
       connections.coaddName: parameters.coaddName
-      # TODO: redundant connection definitions workaround for DM-30210
-      connections.coaddExposures: parameters.template
-      connections.dcrCoadds: dcrCoadd
-      connections.outputSchema: parameters.diaSrcSchema
-      connections.subtractedExposure: parameters.diff
-      connections.scoreExposure: parameters.diffScore
-      connections.warpedExposure: parameters.diffWarp
-      connections.matchedExposure: parameters.diffMatch
-      connections.diaSources: parameters.diaSrcCat
-      connections.inputTemplate: parameters.templateExp
-      # TODO: end DM-30210 workaround
-  imageDifference:
-    class: lsst.pipe.tasks.imageDifference.ImageDifferenceFromTemplateTask
-    config:
-      doWriteWarpedExp: True       # Required for packaging alerts in diaPipe
-      connections.fakesType: parameters.fakesType
-      # TODO: redundant connection definitions workaround for DM-30210
-      connections.coaddName: parameters.coaddName
-      connections.exposure: parameters.fakesPvi
-      connections.coaddExposures: parameters.fakesTemplate
-      connections.dcrCoadds: fakesDcrTemplate
-      connections.outputSchema: parameters.fakesDiaSrcSchema
-      connections.subtractedExposure: parameters.fakesDiff
-      connections.scoreExposure: parameters.fakesDiffScore
-      connections.warpedExposure: parameters.fakesDiffWarp
-      connections.matchedExposure: parameters.fakesDiffMatch
-      connections.diaSources: parameters.fakesDiaSrcCat
-      connections.inputTemplate: parameters.templateExp
-      # TODO: end DM-30210 workaround
-  transformDiaSrcCat:
-    class: lsst.ap.association.TransformDiaSourceCatalogTask
-    config:
-      connections.fakesType: parameters.fakesType
-      # TODO: redundant connection definitions workaround for DM-30210
-      connections.coaddName: parameters.coaddName
-      connections.diaSourceSchema: parameters.fakesDiaSrcSchema
-      connections.diaSourceCat: parameters.fakesDiaSrcCat
-      connections.diffIm: parameters.fakesDiff
-      connections.diaSourceTable: parameters.fakesDiaSrcParquet
-      # TODO: end DM-30210 workaround
-  # Can't have separate with- and without-fakes runs for diaPipe, because there's only one APDB
   diaPipe:
     class: lsst.ap.association.DiaPipelineTask
     config:
-      doWriteAssociatedSources: True
-      connections.fakesType: parameters.fakesType
-      # TODO: redundant connection definitions workaround for DM-30210
-      connections.coaddName: parameters.coaddName
-      connections.diaSourceTable: parameters.fakesDiaSrcParquet
-      connections.diffIm: parameters.fakesDiff
-      connections.warpedExposure: parameters.fakesDiffWarp
-      connections.associatedDiaSources: parameters.fakesAssocSrc
-      # TODO: end DM-30210 workaround
-  fakesMatcher:
-    class: lsst.pipe.tasks.matchFakes.MatchVariableFakesTask
-    config:
-      matchDistanceArcseconds: 0.1
-      connections.coaddName: parameters.coaddName
-      connections.fakesType: parameters.fakesType
-      # TODO: redundant connection definitions workaround for DM-30210
-      connections.fakeCats: parameters.fakesInput
-      connections.diffIm: parameters.fakesDiff
-      connections.associatedDiaSources: parameters.fakesAssocSrc
-      connections.matchedDiaSources: parameters.fakesMatchedSrc
-      # TODO: end DM-30210 workaround
-  fracDiaSourcesToSciSources:
-    # Uses output of imageDifferenceNoFakes
-    class: lsst.ip.diffim.metrics.FractionDiaSourcesToSciSourcesMetricTask
-  timing_imageDifference:
-    class: lsst.verify.tasks.commonMetrics.TimingMetricTask
-    config:
-      connections.labelName: imageDifferenceNoFakes
-      target: imageDifferenceNoFakes.run
-  timing_imageDifference_astrometer:
-    class: lsst.verify.tasks.commonMetrics.TimingMetricTask
-    config:
-      connections.labelName: imageDifferenceNoFakes
-      target: imageDifferenceNoFakes:astrometer.loadAndMatch
-  timing_imageDifference_register:
-    class: lsst.verify.tasks.commonMetrics.TimingMetricTask
-    config:
-      connections.labelName: imageDifferenceNoFakes
-      target: imageDifferenceNoFakes:register.run
-  timing_imageDifference_subtract:
-    class: lsst.verify.tasks.commonMetrics.TimingMetricTask
-    config:
-      connections.labelName: imageDifferenceNoFakes
-      target: imageDifferenceNoFakes:subtract.subtractExposures
-  timing_imageDifference_detection:
+      doPackageAlerts: True
+  timing_transformDiaSrcCat:
     class: lsst.verify.tasks.commonMetrics.TimingMetricTask
     config:
-      connections.labelName: imageDifferenceNoFakes
-      target: imageDifferenceNoFakes:detection.run
-  timing_imageDifference_measurement:
-    class: lsst.verify.tasks.commonMetrics.TimingMetricTask
-    config:
-      connections.labelName: imageDifferenceNoFakes
-      target: imageDifferenceNoFakes:measurement.run
+      connections.labelName: transformDiaSrcCatWithFakes  # Default is transformDiaSrcCat
+      target: transformDiaSrcCatWithFakes.run
 subsets:
   apPipe:
+    # Extend the subset through imageDifference.
     subset:
       - isr
       - characterizeImage
       - calibrate
       - retrieveTemplate
-      - imageDifferenceNoFakes
-    description: >
-      The AP pipeline without fakes. Only includes processing through
-      image differencing.
-  prepareFakes:
-    subset:
-      - createFakes
-      - coaddFakes
-    description: >
-      Creation of fake sources.
-  apPipeWithFakes:
-    subset:
-      - singleFrameWithFakes    # characterizeImage and calibrate with fakes
-      - retrieveTemplateWithFakes
       - imageDifference
-      - transformDiaSrcCat
-      - diaPipe
-      - fakesMatcher
-    description: >
-      The AP pipeline with fakes. Requires apPipe and prepareFakes subsets.
 contracts:
   # Metric inputs must match pipeline outputs
-  - createFakes.connections.fakesType == coaddFakes.connections.fakesType
-  - createFakes.connections.fakesType == singleFrameWithFakes.connections.fakesType
-  - createFakes.connections.fakesType == imageDifference.connections.fakesType
-  - createFakes.connections.fakesType == diaPipe.connections.fakesType
-  - createFakes.connections.fakesType == fakesMatcher.connections.fakesType
-  - createFakes.connections.fakesType == apFakesCompletenessMag20t22.connections.fakesType
-  - createFakes.connections.fakesType == apFakesCompletenessMag22t24.connections.fakesType
-  - createFakes.connections.fakesType == apFakesCompletenessMag24t26.connections.fakesType
-  - coaddFakes.connections.coaddName == singleFrameWithFakes.connections.coaddName
-  - coaddFakes.connections.coaddName == imageDifference.connections.coaddName
-  - coaddFakes.connections.coaddName == diaPipe.connections.coaddName
-  - coaddFakes.connections.coaddName == fakesMatcher.connections.coaddName
-  - coaddFakes.connections.coaddName == apFakesCompletenessMag20t22.connections.coaddName
-  - coaddFakes.connections.coaddName == apFakesCompletenessMag22t24.connections.coaddName
-  - coaddFakes.connections.coaddName == apFakesCompletenessMag24t26.connections.coaddName
-  - imageDifferenceNoFakes.connections.coaddName == fracDiaSourcesToSciSources.connections.coaddName
-  - imageDifferenceNoFakes.connections.fakesType == fracDiaSourcesToSciSources.connections.fakesType
+  # Use of ConnectionsClass for templated fields is a workaround for DM-30210
+  - fakesMatch.connections.ConnectionsClass(config=fakesMatch).matchedDiaSources.name ==
+      apFakesCompletenessMag20t22.connections.ConnectionsClass(config=apFakesCompletenessMag20t22).matchedFakes.name
+  - fakesMatch.connections.ConnectionsClass(config=fakesMatch).matchedDiaSources.name ==
+      apFakesCompletenessMag22t24.connections.ConnectionsClass(config=apFakesCompletenessMag22t24).matchedFakes.name
+  - fakesMatch.connections.ConnectionsClass(config=fakesMatch).matchedDiaSources.name ==
+      apFakesCompletenessMag24t26.connections.ConnectionsClass(config=apFakesCompletenessMag24t26).matchedFakes.name
+  - imageDifference.connections.ConnectionsClass(config=imageDifference).diaSources.name ==
+      fracDiaSourcesToSciSources.connections.ConnectionsClass(config=fracDiaSourcesToSciSources).diaSources.name

pipelines/DarkEnergyCamera/ApVerify.yaml

@@ -18,5 +18,6 @@ tasks:
 contracts:
   # Must re-declare contracts that cross apPipe and metrics boundary, as
   # these were removed on import.
-  - imageDifference.connections.coaddName == fracDiaSourcesToSciSources.connections.coaddName
-  - imageDifference.connections.fakesType == fracDiaSourcesToSciSources.connections.fakesType
+  # Use of ConnectionsClass for templated fields is a workaround for DM-30210
+  - imageDifference.connections.ConnectionsClass(config=imageDifference).diaSources.name ==
+      fracDiaSourcesToSciSources.connections.ConnectionsClass(config=fracDiaSourcesToSciSources).diaSources.name

pipelines/DarkEnergyCamera/ApVerifyWithFakes.yaml

@@ -8,9 +8,9 @@ instrument: lsst.obs.decam.DarkEnergyCamera
 description: Fully instrumented AP pipeline with fakes, specialized for DECam
 imports:
   - location: $AP_VERIFY_DIR/pipelines/ApVerifyWithFakes.yaml
-  # Can't use $AP_PIPE_DIR/pipelines/DarkEnergyCamera/ApPipe.yaml here because
-  # ApVerifyWithFakes changes task labels in order to set up parallel fakes
-  # and non-fakes pipelines.
+  # Can't use $AP_PIPE_DIR/pipelines/DarkEnergyCamera/ApPipeWithFakes.yaml here
+  # because the changes made by that file and ../ApVerifyWithFakes.yaml are
+  # hard to separate.
 tasks:
   isr:
     class: lsst.ip.isr.IsrTask
@@ -24,7 +24,7 @@ tasks:
     config:
       photoCal.match.referenceSelection.magLimit.fluxField: i_flux
       photoCal.match.referenceSelection.magLimit.maximum: 22.0
-  singleFrameWithFakes:
+  processVisitFakes:
     class: lsst.pipe.tasks.processCcdWithFakes.ProcessCcdWithVariableFakesTask
     config:
       calibrate.photoCal.match.referenceSelection.magLimit.fluxField: i_flux

pipelines/HyperSuprimeCam/ApVerify.yaml

@@ -18,5 +18,6 @@ tasks:
 contracts:
   # Must re-declare contracts that cross apPipe and metrics boundary, as
   # these were removed on import.
-  - imageDifference.connections.coaddName == fracDiaSourcesToSciSources.connections.coaddName
-  - imageDifference.connections.fakesType == fracDiaSourcesToSciSources.connections.fakesType
+  # Use of ConnectionsClass for templated fields is a workaround for DM-30210
+  - imageDifference.connections.ConnectionsClass(config=imageDifference).diaSources.name ==
+      fracDiaSourcesToSciSources.connections.ConnectionsClass(config=fracDiaSourcesToSciSources).diaSources.name

pipelines/HyperSuprimeCam/ApVerifyWithFakes.yaml

@@ -5,6 +5,6 @@ instrument: lsst.obs.subaru.HyperSuprimeCam
 description: Fully instrumented AP pipeline with fakes, specialized for HSC
 imports:
   - location: $AP_VERIFY_DIR/pipelines/ApVerifyWithFakes.yaml
-  # Can't use $AP_PIPE_DIR/pipelines/HyperSuprimeCam/ApPipe.yaml here because
-  # ApVerifyWithFakes changes task labels in order to set up parallel fakes
-  # and non-fakes pipelines.
+  # Can't use $AP_PIPE_DIR/pipelines/HyperSuprimeCam/ApPipeWithFakes.yaml here
+  # because the changes made by that file and ../ApVerifyWithFakes.yaml are
+  # hard to separate.

pipelines/MetricsForFakes.yaml

@@ -12,9 +12,6 @@ tasks:
       magMax: 22
       connections.coaddName: parameters.coaddName
       connections.fakesType: parameters.fakesType
-      # TODO: redundant connection definitions workaround for DM-30210
-      connections.matchedFakes: parameters.fakesMatchedSrc
-      # TODO: end DM-30210 workaround
   apFakesCompletenessMag22t24:
     class: lsst.ap.pipe.metrics.ApFakesCompletenessMetricTask
     config:
@@ -23,9 +20,6 @@ tasks:
       magMax: 24
       connections.coaddName: parameters.coaddName
       connections.fakesType: parameters.fakesType
-      # TODO: redundant connection definitions workaround for DM-30210
-      connections.matchedFakes: parameters.fakesMatchedSrc
-      # TODO: end DM-30210 workaround
   apFakesCompletenessMag24t26:
     class: lsst.ap.pipe.metrics.ApFakesCompletenessMetricTask
     config:
@@ -34,9 +28,6 @@ tasks:
       magMax: 26
       connections.coaddName: parameters.coaddName
       connections.fakesType: parameters.fakesType
-      # TODO: redundant connection definitions workaround for DM-30210
-      connections.matchedFakes: parameters.fakesMatchedSrc
-      # TODO: end DM-30210 workaround
   apFakesCountMag20t22:
     class: lsst.ap.pipe.metrics.ApFakesCountMetricTask
     config:
@@ -45,9 +36,6 @@ tasks:
       magMax: 22
       connections.coaddName: parameters.coaddName
       connections.fakesType: parameters.fakesType
-      # TODO: redundant connection definitions workaround for DM-30210
-      connections.matchedFakes: parameters.fakesMatchedSrc
-      # TODO: end DM-30210 workaround
   apFakesCountMag22t24:
     class: lsst.ap.pipe.metrics.ApFakesCountMetricTask
     config:
@@ -56,9 +44,6 @@ tasks:
       magMax: 24
       connections.coaddName: parameters.coaddName
       connections.fakesType: parameters.fakesType
-      # TODO: redundant connection definitions workaround for DM-30210
-      connections.matchedFakes: parameters.fakesMatchedSrc
-      # TODO: end DM-30210 workaround
   apFakesCountMag24t26:
     class: lsst.ap.pipe.metrics.ApFakesCountMetricTask
     config:
@@ -67,9 +52,6 @@ tasks:
       magMax: 26
       connections.coaddName: parameters.coaddName
       connections.fakesType: parameters.fakesType
-      # TODO: redundant connection definitions workaround for DM-30210
-      connections.matchedFakes: parameters.fakesMatchedSrc
-      # TODO: end DM-30210 workaround
   apFakesCount:
     class: lsst.ap.pipe.metrics.ApFakesCountMetricTask
     config:
@@ -78,6 +60,3 @@ tasks:
       magMax: 39
       connections.coaddName: parameters.coaddName
       connections.fakesType: parameters.fakesType
-      # TODO: redundant connection definitions workaround for DM-30210
-      connections.matchedFakes: parameters.fakesMatchedSrc
-      # TODO: end DM-30210 workaround

pipelines/MetricsMisc.yaml

@@ -30,6 +30,8 @@ tasks:
     class: lsst.ip.diffim.metrics.NumberSciSourcesMetricTask
   fracDiaSourcesToSciSources:
     class: lsst.ip.diffim.metrics.FractionDiaSourcesToSciSourcesMetricTask
+    config:
+      connections.coaddName: parameters.coaddName
   numDeblendedSciSources:
     class: lsst.pipe.tasks.metrics.NumberDeblendedSourcesMetricTask
   numDeblendChildSciSources: