Merge pull request #359 from lsst/tickets/DM-29776

DM-29776: Gen3 DRP pipeline updates and reorganization

pipelines/DRP.yaml

@@ -12,6 +12,14 @@ tasks:
     # RFC-approve a better home for it.
     class: lsst.jointcal.JointcalTask
     config:
+      # Disable photometry because we don't need it (FGCM takes care of that)
+      # and it's broken in Gen3 (DM-29884).  Also disable color terms because
+      # jointcal warns if that's true but doPhotometry=False.  It would be
+      # better to disable this via config (DM-29885), but it's hard to make
+      # that compatible with the jointcal tests because jointcal depends on
+      # obs_subaru.
+      doPhotometry: false
+      applyColorTerms: false
       python: |
         # This override is temporary until DM-29008.
         from lsst.utils import getPackageDir
@@ -103,14 +111,76 @@ subsets:
       - fgcmOutputProducts
     description: >
       Subset that includes all FGCM tasks.
+  singleFrame:
+    subset:
+      - isr
+      - characterizeImage
+      - calibrate
+      - writeSourceTable
+      - transformSourceTable
+      - consolidateSourceTable
+      - consolidateVisitSummary
+      - skyCorr
+    description: >
+      Set of tasks for complete single frame processing. Analogous to
+      SingleFrameDriver.
   multiVisit:
+    subset: []
+    description: |
+      The multiVisit subset defined in pipe_tasks' DRP.yaml is not safe to
+      use on HSC for various reasons; use 'step1', 'step2', and 'step3' subsets
+      instead.  It may be re-enabled in the future.
+  step1:
     subset:
+      - isr
+      - characterizeImage
+      - calibrate
+      - writeSourceTable
+      - transformSourceTable
+      - consolidateSourceTable
+      - consolidateVisitSummary
+      - skyCorr
       - fgcmBuildStarsTable
       - fgcmFitCycle
       - fgcmOutputProducts
+    description: |
+      Tasks that can be run together to start the DRP pipeline.
+
+      These should never be run with 'tract' or 'patch' as part of the data ID
+      expression if any later steps will also be run, because downstream steps
+      require full visits and 'tract' and 'patch' constraints will always
+      select partial visits that overlap that region.
+
+      This includes FGCM because it's configured here to run in "global" mode,
+      which means one should not use 'tract' expression to constrain it, and if
+      one _did_ run it with a tract constraint (which would be a common
+      occurrence if it was included in any later step), it would be fed the
+      wrong (partial-visit) inputs to its 'background' connection.
+
+      This subset is considered a workaround for missing middleware and task
+      functionality.  It may be removed in the future.
+  step2:
+    subset:
       - jointcal
       - makeWarp
       - assembleCoadd
+    description: >
+      Tasks that can be run together, but only after the 'step1'.
+
+      These should be run with explicit 'tract' constraints essentially all the
+      time, because otherwise quanta will be created for jobs with only partial
+      visit coverage.
+
+      This subset cannot be run at the same time as the tasks in the 'step3'
+      at present because assembleCoadd has no good way to communicate to
+      downstream tasks that there is no coadd for a particular patch+band
+      combination because there no input data.  Creating a new QuantumGraph
+      after all possible coadds have been built works around this.
+
+      This subset is considered a workaround for missing middleware and task
+      functionality.  It may be removed in the future.
+  step3:
+    subset:
       - detection
       - mergeDetections
       - deblend
@@ -122,8 +192,18 @@ subsets:
       - writeObjectTable
       - consolidateObjectTable
     description: >
-      All tasks that process multiple visits together.
-      This subset plus singleFrame should generally be equivalent to the full
-      pipeline.  At present this is not true, because diffimDRP would benefit
-      from more testing before we start running it with everything else by
-      default.
+      Tasks that can be run together, but only after the 'step1' and 'step2'
+      subsets.
+
+      Adding a 'tract' constraint to the data ID expression when running this
+      subset should have no effect on the result, if 'step2' was run with a
+      tract constraint.
+
+      It is expected that many forcedPhotCcd quanta will "normally" fail when
+      running this subset, but this isn't a problem right now because there
+      are no tasks downstream of it.  If other tasks regularly fail or we add
+      tasks downstream of forcedPhotCcd, these subsets or the tasks will need
+      additional changes.
+
+      This subset is considered a workaround for missing middleware and task
+      functionality.  It may be removed in the future.