DM-35551: restructure and clean up pipelines

ingredients/DECam/DRP.yaml

@@ -7,9 +7,10 @@ description: |
   pre-generated for DECam data processing runs.
 instrument: lsst.obs.decam.DarkEnergyCamera
 imports:
-  - location: $DRP_PIPE_DIR/ingredients/DRP.yaml
+  - location: $DRP_PIPE_DIR/ingredients/DRP-full.yaml
     exclude:
-      - writeSourceTable  # cannot reconfigure because different dimensions.
+      - skyCorr
+      - fgcm
 tasks:
   isrForCrosstalkSources:
     class: lsst.ip.isr.IsrTask
@@ -37,30 +38,11 @@ tasks:
     config:
       photoCal.match.referenceSelection.magLimit.fluxField: i_flux
       photoCal.match.referenceSelection.magLimit.maximum: 22.0
-  writePreSourceTable:
-    class: lsst.pipe.tasks.postprocess.WriteSourceTableTask
-    config:
-      connections.outputCatalog: preSource
-  transformPreSourceTable:
-    class: lsst.pipe.tasks.postprocess.TransformSourceTableTask
-    config:
-      connections.inputCatalog: preSource
-      connections.outputCatalog: preSourceTable
-  consolidatePreSourceTable:
-    class: lsst.pipe.tasks.postprocess.ConsolidateSourceTableTask
-    config:
-      connections.inputCatalogs: preSourceTable
-      connections.outputCatalog: preSourceTable_visit
-  jointcal:
-    class: lsst.jointcal.JointcalTask
-    config:
-      connections.inputSourceTableVisit: preSourceTable_visit
   writeRecalibratedSourceTable:
     class: lsst.pipe.tasks.postprocess.WriteRecalibratedSourceTableTask
     config:
       useGlobalExternalPhotoCalib: false
       connections.photoCalibName: "jointcal"
-      connections.outputCatalog: source
   makeWarp:
     class: lsst.pipe.tasks.makeCoaddTempExp.MakeWarpTask
     config:

ingredients/DRP-full.yaml

@@ -0,0 +1,78 @@
+description:
+  Base Pipeline for Data Release Production process, with full multi-epoch
+  calibration and characterizations.
+
+  Never run this pipeline or any of its subsets directly; always use those in
+  the 'pipelines' directory (or create a custom one) instead.
+imports:
+  - location: $DRP_PIPE_DIR/ingredients/DRP-minimal-calibration.yaml
+    exclude:
+      - writeSourceTable  # cannot reconfigure because different dimensions.
+tasks:
+  skyCorr:
+    class: lsst.pipe.drivers.skyCorrection.SkyCorrectionTask
+  isolatedStarAssociation:
+    class: lsst.pipe.tasks.isolatedStarAssociation.IsolatedStarAssociationTask
+    config:
+      connections.source_table_visit: preSourceTable_visit
+  jointcal:
+    class: lsst.jointcal.JointcalTask
+    config:
+      connections.inputSourceTableVisit: preSourceTable_visit
+  fgcmBuildStarsTable:
+    class: lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableTask
+    config:
+      connections.sourceTable_visit: preSourceTable_visit
+  fgcmFitCycle:
+    class: lsst.fgcmcal.fgcmFitCycle.FgcmFitCycleTask
+  fgcmOutputProducts:
+    class: lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask
+  writePreSourceTable:
+    class: lsst.pipe.tasks.postprocess.WriteSourceTableTask
+    config:
+      connections.outputCatalog: preSource
+  transformPreSourceTable:
+    class: lsst.pipe.tasks.postprocess.TransformSourceTableTask
+    config:
+      connections.inputCatalog: preSource
+      connections.outputCatalog: preSourceTable
+  consolidatePreSourceTable:
+    class: lsst.pipe.tasks.postprocess.ConsolidateSourceTableTask
+    config:
+      connections.inputCatalogs: preSourceTable
+      connections.outputCatalog: preSourceTable_visit
+  writeRecalibratedSourceTable:
+    class: lsst.pipe.tasks.postprocess.WriteRecalibratedSourceTableTask
+    config:
+      connections.outputCatalog: source
+  measure:
+    class: lsst.pipe.tasks.multiBand.MeasureMergedCoaddSourcesTask
+    config:
+      connections.sourceTableHandles: preSourceTable_visit
+subsets:
+  # This pipeline file only defines subsets that have a consistent or
+  # near-consistent definition as a set of tasks, for use by downstream
+  # pipelines as way to include or exclude those tasks.  It does not (and
+  # should not) contain "step" subsets aimed at splitting up a processing
+  # campaign.
+  fgcm:
+    subset:
+      - fgcmBuildStarsTable
+      - fgcmFitCycle
+      - fgcmOutputProducts
+    description: |
+      Subset that includes all FGCM tasks.
+  preSourceTable:
+    subset:
+      - writePreSourceTable
+      - transformPreSourceTable
+      - consolidatePreSourceTable
+    description: >
+      Set of tasks to generate parquet PreSource Tables, from single-visit
+      processing steps only.
+  sourceTable:
+    subset:
+      - writeRecalibratedSourceTable
+    description: >
+      Set of tasks to generate parquet Source Tables, using final
+      calibrations.

ingredients/DRP.yaml → ingredients/DRP-minimal-calibration.yaml

@@ -1,5 +1,6 @@
 description: |
-  Base Pipeline that describes the Data Release Production process.
+  Base Pipeline for Data Release Production process, with only simple
+  single-epoch calibration and characterizations.
 
   Never run this pipeline or any of its subsets directly; always use those in
   the 'pipelines' directory (or create a custom one) instead.
@@ -144,6 +145,11 @@ tasks:
       connections.inputCatalogs: forcedSourceOnDiaObjectTable
       connections.outputCatalog: forcedSourceOnDiaObjectTable_tract
 subsets:
+  # This pipeline file only defines subsets that have a consistent or
+  # near-consistent definition as a set of tasks, for use by downstream
+  # pipelines as way to include or exclude those tasks.  It does not (and
+  # should not) contain "step" subsets aimed at splitting up a processing
+  # campaign.
   processCcd:
     subset:
       - isr
@@ -160,22 +166,6 @@ subsets:
     description: >
       Set of tasks to generate parquet Source Tables from output of processCcd
       subset.
-  singleFrame:
-    subset:
-      - isr
-      - characterizeImage
-      - calibrate
-      - writeSourceTable
-      - transformSourceTable
-      - consolidateSourceTable
-      - consolidateVisitSummary
-      - isolatedStarAssociation
-      - finalizeCharacterization
-      - makeCcdVisitTable
-      - makeVisitTable
-    description: >
-      Set of tasks for complete single frame processing. Analogous to
-      SingleFrameDriver.
   coaddition:
     subset:
       - makeWarp
@@ -207,28 +197,6 @@ subsets:
       - forcedPhotCoadd
     description: >
       A set of tasks to run when doing forced measurements.
-  multiVisit:
-    subset:
-      - makeWarp
-      - assembleCoadd
-      - healSparsePropertyMaps
-      - detection
-      - mergeDetections
-      - deblend
-      - measure
-      - mergeMeasurements
-      - forcedPhotCcd
-      - forcedPhotCoadd
-      - transformObjectTable
-      - writeObjectTable
-      - consolidateObjectTable
-    description: >
-      All tasks that process multiple visits together.
-
-      This subset plus singleFrame should generally be equivalent to the full
-      pipeline.  At present this is not true, because diffimDRP would benefit
-      from more testing before we start running it with everything else by
-      default.
   diffimDRP:
     subset:
       - selectGoodSeeingVisits

ingredients/HSC/DRP+fakes.yaml

@@ -0,0 +1,145 @@
+description: The DRP Fakes pipeline specalised for the HSC RC2 dataset.
+instrument: lsst.obs.subaru.HyperSuprimeCam
+imports:
+  location: "$DRP_PIPE_DIR/ingredients/HSC/DRP.yaml"
+  exclude:
+    - skyCorr
+    - jointcal
+    - fgcm
+    - fgcmBuildStarsTable
+    - fgcmFitCycle
+    - fgcmOutputProducts
+    - isr
+    - characterizeImage
+    - calibrate
+    - selectGoodSeeingVisits
+    - templateGen
+    - imageDifference
+    - transformDiaSourceCat
+    - consolidateDiaSourceTable
+    - drpAssociation
+    - drpDiaCalculation
+    - forcedPhotDiffim
+    - forcedPhotCcdOnDiaObjects
+    - forcedPhotDiffOnDiaObjects
+    - transformForcedSourceTable
+    - consolidateForcedSourceTable
+    - getTemplate
+    - writeForcedSourceTable
+    - consolidateAssocDiaSourceTable
+    - consolidateFullDiaObjectTable
+    - writeForcedSourceOnDiaObjectTable
+    - transformForcedSourceOnDiaObjectTable
+    - consolidateForcedSourceOnDiaObjectTable
+    - faro_all
+tasks:
+  singleFrameWithFakes:
+    class: lsst.pipe.tasks.processCcdWithFakes.ProcessCcdWithFakesTask
+    config:
+      insertFakes.doProcessAllDataIds: true
+      connections.photoCalibName: "fgcm"
+      doApplyExternalGlobalPhotoCalib: true
+      doApplyExternalTractSkyWcs: true
+  writePreSourceTable:
+    class: lsst.pipe.tasks.postprocess.WriteSourceTableTask
+    config:
+      connections.catalogType: "fakes_"
+      connections.outputCatalog: fakes_preSource
+  transformPreSourceTable:
+    class: lsst.pipe.tasks.postprocess.TransformSourceTableTask
+    config:
+      connections.inputCatalog: fakes_preSource
+      connections.outputCatalog: fakes_preSourceTable
+  consolidatePreSourceTable:
+    class: lsst.pipe.tasks.postprocess.ConsolidateSourceTableTask
+    config:
+      connections.inputCatalogs: fakes_preSourceTable
+      connections.outputCatalog: fakes_preSourceTable_visit
+  writeRecalibratedSourceTable:
+    class: lsst.pipe.tasks.postprocess.WriteRecalibratedSourceTableTask
+    config:
+      connections.outputCatalog: fakes_source
+      connections.catalogType: "fakes_"
+  transformSourceTable:
+    class: lsst.pipe.tasks.postprocess.TransformSourceTableTask
+    config:
+      connections.catalogType: "fakes_"
+  consolidateSourceTable:
+    class: lsst.pipe.tasks.postprocess.ConsolidateSourceTableTask
+    config:
+      connections.catalogType: "fakes_"
+  consolidateVisitSummary:
+    class: lsst.pipe.tasks.postprocess.ConsolidateVisitSummaryTask
+    config:
+      connections.calexpType: "fakes_"
+  makeCcdVisitTable:
+    class: lsst.pipe.tasks.postprocess.MakeCcdVisitTableTask
+    config:
+      connections.calexpType: "fakes_"
+  makeVisitTable:
+    class: lsst.pipe.tasks.postprocess.MakeVisitTableTask
+    config:
+      connections.calexpType: "fakes_"
+  assembleCoadd:
+    class: lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask
+    config:
+      connections.inputCoaddName: "fakes_deep"
+      connections.outputCoaddName: "fakes_deep"
+  makeWarp:
+    class: lsst.pipe.tasks.makeCoaddTempExp.MakeWarpTask
+    config:
+      connections.photoCalibName: "fgcm"
+      useGlobalExternalPhotoCalib: true
+      connections.coaddName: "fakes_deep"
+      connections.calexpType: "fakes_"
+  detection:
+    class: lsst.pipe.tasks.multiBand.DetectCoaddSourcesTask
+    config:
+      connections.inputCoaddName: "fakes_deep"
+      connections.outputCoaddName: "fakes_deep"
+  mergeDetections:
+    class: lsst.pipe.tasks.mergeDetections.MergeDetectionsTask
+    config:
+      connections.inputCoaddName: "fakes_deep"
+      connections.outputCoaddName: "fakes_deep"
+  deblend:
+    class: lsst.pipe.tasks.deblendCoaddSourcesPipeline.DeblendCoaddSourcesMultiTask
+    config:
+      connections.inputCoaddName: "fakes_deep"
+      connections.outputCoaddName: "fakes_deep"
+  measure:
+    class: lsst.pipe.tasks.multiBand.MeasureMergedCoaddSourcesTask
+    config:
+      connections.inputCoaddName: "fakes_deep"
+      connections.outputCoaddName: "fakes_deep"
+  mergeMeasurements:
+    class: lsst.pipe.tasks.mergeMeasurements.MergeMeasurementsTask
+    config:
+      connections.inputCoaddName: "fakes_deep"
+      connections.outputCoaddName: "fakes_deep"
+  writeObjectTable:
+    class: lsst.pipe.tasks.postprocess.WriteObjectTableTask
+    config:
+      connections.coaddName: "fakes_deep"
+  transformObjectTable:
+    class: lsst.pipe.tasks.postprocess.TransformObjectCatalogTask
+    config:
+      connections.coaddName: "fakes_deep"
+  forcedPhotCcd:
+    class: lsst.meas.base.forcedPhotCcd.ForcedPhotCcdTask
+    config:
+      connections.inputCoaddName: "fakes_deep"
+  forcedPhotCoadd:
+    class: lsst.meas.base.forcedPhotCoadd.ForcedPhotCoaddTask
+    config:
+      connections.inputCoaddName: "fakes_deep"
+      connections.outputCoaddName: "fakes_deep"
+  healSparsePropertyMaps:
+    class: lsst.pipe.tasks.healSparseMapping.HealSparsePropertyMapTask
+    config:
+      connections.coaddName: "fakes_deep"
+      connections.calexpType: "fakes_"
+  consolidateHealSparsePropertyMaps:
+    class: lsst.pipe.tasks.healSparseMapping.ConsolidateHealSparsePropertyMapTask
+    config:
+      connections.coaddName: "fakes_deep"