Resolve conflicts

attachments/DRP-RC2.yaml

@@ -0,0 +1,332 @@
+description: The DRP pipeline specialized for the HSC RC2 dataset.
+instrument: lsst.obs.subaru.HyperSuprimeCam
+imports:
+  - $DRP_PIPE_DIR/ingredients/DRP.yaml
+  - $FARO_DIR/pipelines/metrics_pipeline_jointcal_fgcm.yaml
+tasks:
+  skyCorr:
+    class: lsst.pipe.drivers.skyCorrection.SkyCorrectionTask
+  jointcal:
+    # Running jointcal is something we want to do for all real-world
+    # instruments, so it doesn't belong in an HSC-specific pipeline, let alone
+    # an RC2-specific one.  We just haven't done the pipeline refactoring that
+    # would fix this.
+    class: lsst.jointcal.JointcalTask
+  # The FGCM configurations below are appropriate specifically for RC2; this is
+  # not a bad place for it, because we don't have (and may never have)
+  # configuration for FGCM that works across arbitrary datasets; it depends too
+  # much on what filters are in play.  But it still might make sense to
+  # refactor the configurations below into separate (but still RC2-specific)
+  # config files.
+  fgcmBuildStarsTable:
+    class: lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableTask
+    config:
+      minPerBand: 1
+      densityCutMaxPerPixel: 1500
+      primaryBands: ["i", "r", "g", "z", "y", "N921"]
+      doReferenceMatches: true
+      randomSeed: 23456
+  fgcmFitCycle:
+    class: lsst.fgcmcal.fgcmFitCycle.FgcmFitCycleTask
+    config:
+      doMultipleCycles: true
+      multipleCyclesFinalCycleNumber: 4
+      randomSeed: 89234
+      outfileBase: "fgcmFitCycleRC2"
+      bands: ["g", "r", "i", "z", "N921", "y"]
+      fitBands: ["g", "r", "i", "z", "N921", "y"]
+      maxIterBeforeFinalCycle: 50
+      expGrayPhotometricCutDict:
+        g: -0.05
+        r: -0.05
+        i: -0.05
+        z: -0.05
+        N921: -0.05
+        y: -0.05
+      expGrayHighCutDict:
+        g: 0.10
+        r: 0.10
+        i: 0.10
+        z: 0.10
+        N921: 0.10
+        y: 0.10
+      aperCorrFitNBins: 0
+      minStarPerExp: 100
+      expVarGrayPhotometricCutDict:
+        g: 0.0025
+        r: 0.0025
+        i: 0.0025
+        z: 0.0025
+        N921: 0.005
+        y: 0.0025
+      minExpPerNight: 3
+      python: |
+        from lsst.fgcmcal import Sedterm, Sedboundaryterm
+        config.sedboundaryterms.data = {'gr': Sedboundaryterm(primary='g', secondary='r'),
+                                        'ri': Sedboundaryterm(primary='r', secondary='i'),
+                                        'iz': Sedboundaryterm(primary='i', secondary='z'),
+                                        'zy': Sedboundaryterm(primary='z', secondary='y'),
+                                        'N921z': Sedboundaryterm(primary='N921', secondary='z')}
+        config.sedterms.data = {'g': Sedterm(primaryTerm='gr', secondaryTerm='ri', constant=1.6),
+                                'r': Sedterm(primaryTerm='gr', secondaryTerm='ri', constant=0.9),
+                                'i': Sedterm(primaryTerm='ri', secondaryTerm='iz', constant=1.0),
+                                'z': Sedterm(primaryTerm='iz', secondaryTerm='zy', constant=1.0),
+                                'y': Sedterm(primaryTerm='zy', secondaryTerm='iz', constant=0.25,
+                                             extrapolated=True, primaryBand='y', secondaryBand='z',
+                                             tertiaryBand='i'),
+                                'N921': Sedterm(primaryTerm='N921z', constant=0.5)}
+  fgcmOutputProducts:
+    class: lsst.fgcmcal.fgcmOutputProducts.FgcmOutputProductsTask
+    config:
+      doRefcatOutput: false
+      cycleNumber: 4
+  forcedPhotCcd:
+    class: lsst.meas.base.forcedPhotCcd.ForcedPhotCcdTask
+    config:
+      # The application of external calibrations doesn't work with Gen3 now (DM-17062)
+      doApplyExternalPhotoCalib: false
+      doApplyExternalSkyWcs: false
+      doApplySkyCorr: false
+subsets:
+  fgcm:
+    subset:
+      - fgcmBuildStarsTable
+      - fgcmFitCycle
+      - fgcmOutputProducts
+    description: >
+      Subset that includes all FGCM tasks.
+  singleFrame:
+    subset:
+      - isr
+      - characterizeImage
+      - calibrate
+      - writeSourceTable
+      - transformSourceTable
+      - consolidateSourceTable
+      - consolidateVisitSummary
+      - skyCorr
+    description: >
+      Set of tasks for complete single frame processing. Analogous to
+      SingleFrameDriver.
+  multiVisit:
+    subset: []
+    description: >
+      The multiVisit subset defined in pipe_tasks' DRP.yaml is not safe to
+      use on HSC for various reasons; use 'step1', 'step2', and 'step3' subsets
+      instead.  It may be re-enabled in the future.
+  faro_step2:
+    subset:
+      - nsrcMeasVisit
+      - TE3
+      - TE4
+    description: >
+      Set of tasks for calculation of metrics via faro. This includes tasks
+      that operate on single visits, and thus appear in step2 below.
+  faro_step3:
+    subset:
+      - matchCatalogsTract
+      - matchCatalogsPatch
+      - matchCatalogsPatchMultiBand
+      - PA1
+      - PF1_design
+      - AM1
+      - AM2
+      - AM3
+      - AD1_design
+      - AD2_design
+      - AD3_design
+      - AF1_design
+      - AF2_design
+      - AF3_design
+      - AB1
+      - modelPhotRepGal1
+      - modelPhotRepGal2
+      - modelPhotRepGal3
+      - modelPhotRepGal4
+      - modelPhotRepStar1
+      - modelPhotRepStar2
+      - modelPhotRepStar3
+      - modelPhotRepStar4
+      - psfPhotRepStar1
+      - psfPhotRepStar2
+      - psfPhotRepStar3
+      - psfPhotRepStar4
+    description: >
+      Set of tasks for calculation of metrics via faro. This includes tasks
+      that operate on matched visits, and thus appear in step3 below.
+  faro_step5:
+    subset:
+      - TE1
+      - TE2
+      - wPerp
+    description: >
+      Set of tasks for calculation of metrics via faro. This includes tasks
+      that operate on coadds, and thus appear in step5 below.
+  step1:
+    subset:
+      - isr
+      - characterizeImage
+      - calibrate
+      - writeSourceTable
+      - transformSourceTable
+    description: >
+      Per-detector tasks that can be run together to start the DRP pipeline.
+
+      These should never be run with 'tract' or 'patch' as part of the data ID
+      expression if any later steps will also be run, because downstream steps
+      require full visits and 'tract' and 'patch' constraints will always
+      select partial visits that overlap that region.
+  step2:
+    subset:
+      - consolidateSourceTable
+      - consolidateVisitSummary
+      - skyCorr
+      - fgcmBuildStarsTable
+      - fgcmFitCycle
+      - fgcmOutputProducts
+      - nsrcMeasVisit
+      - TE3
+      - TE4
+    description: >
+      Per-visit tasks that can be run together, but only after the 'step1'.
+
+      These should never be run with 'tract' or 'patch' as part of the data ID
+      expression. skyCorr and FGCM require full visits and 'tract' and 'patch'
+      constraints will always select partial visits that overlap that region.
+
+      This includes FGCM because it's configured here to run in "global" mode,
+      which means one should not use 'tract' expression to constrain it, and if
+      one _did_ run it with a tract constraint (which would be a common
+      occurrence if it was included in any later step), it would be fed the
+      wrong (partial-visit) inputs to its 'background' connection.
+
+      This subset is considered a workaround for missing middleware and task
+      functionality.  It may be removed in the future.
+  step3:
+    subset:
+      - jointcal
+      - makeWarp
+      - assembleCoadd
+      - detection
+      - mergeDetections
+      - deblend
+      - measure
+      - mergeMeasurements
+      - forcedPhotCoadd
+      - transformObjectTable
+      - writeObjectTable
+      - consolidateObjectTable
+      - healSparsePropertyMaps
+      - selectGoodSeeingVisits
+      - templateGen
+      - matchCatalogsTract
+      - matchCatalogsPatch
+      - matchCatalogsPatchMultiBand
+      - PA1
+      - PF1_design
+      - AM1
+      - AM2
+      - AM3
+      - AD1_design
+      - AD2_design
+      - AD3_design
+      - AF1_design
+      - AF2_design
+      - AF3_design
+      - AB1
+      - modelPhotRepGal1
+      - modelPhotRepGal2
+      - modelPhotRepGal3
+      - modelPhotRepGal4
+      - modelPhotRepStar1
+      - modelPhotRepStar2
+      - modelPhotRepStar3
+      - modelPhotRepStar4
+      - psfPhotRepStar1
+      - psfPhotRepStar2
+      - psfPhotRepStar3
+      - psfPhotRepStar4
+    description: >
+      Tasks that can be run together, but only after the 'step1' and 'step2'
+      subsets.
+
+      These should be run with explicit 'tract' constraints essentially all the
+      time, because otherwise quanta will be created for jobs with only partial
+      visit coverage.
+
+      It is expected that many forcedPhotCcd quanta will "normally" fail when
+      running this subset, but this isn't a problem right now because there are
+      no tasks downstream of it.  If other tasks regularly fail or we add tasks
+      downstream of forcedPhotCcd, these subsets or the tasks will need
+      additional changes.
+
+      This subset is considered a workaround for missing middleware and task
+      functionality.  It may be removed in the future.
+  step4:
+    subset:
+      - forcedPhotCcd
+      - forcedPhotDiffim
+      - getTemplate
+      - imageDifference
+      - transformDiaSourceCat
+      - writeForcedSourceTable
+    description: >
+      Tasks that can be run together, but only after the 'step1', 'step2' and
+      'step3' subsets
+
+      These detector-level tasks should not be run with 'tract' or 'patch' as
+      part of the data ID expression if all reference catalogs or diffIm
+      templates that cover these detector-level quanta are desired.
+  step5:
+    subset:
+      - drpAssociation
+      - drpDiaCalculation
+      - forcedPhotCcdOnDiaObjects
+      - forcedPhotDiffOnDiaObjects
+      - transformForcedSourceTable
+      - consolidateForcedSourceTable
+      - consolidateAssocDiaSourceTable
+      - consolidateFullDiaObjectTable
+      - writeForcedSourceOnDiaObjectTable
+      - transformForcedSourceOnDiaObjectTable
+      - consolidateForcedSourceOnDiaObjectTable
+      - TE1
+      - TE2
+      - wPerp
+    description: >
+      Tasks that can be run together, but only after the 'step1', 'step2',
+      'step3', and 'step4' subsets
+
+      This step includes patch-level aggregation Tasks. These should be run
+      with explicit 'tract' constraints in the data query, otherwise quanta
+      will be created for jobs with only partial visit coverage.
+      'consolidateForcedSourceTable' is a tract-level task that aggregates
+      patches and should be rerun if any of the patches fail.
+  step6:
+    subset:
+      - consolidateDiaSourceTable
+    description: >
+      Tasks that can be run together, but only after the 'step1', 'step2',
+      'step3', and 'step4' subsets
+
+      This step includes visit-level aggregation tasks. Running without tract
+      or patch in the data query is recommended, otherwise the outputs of
+      consolidateDiaSourceTable will not contain complete visits.
+
+      This subset is separate from step4 to signal to operators to pause to
+      assess unexpected image differencing failures before these aggregation
+      steps. Otherwise, if run in the same quantum graph, aggregated data
+      products (e.g. diaObjects) would not be created if one or more of the
+      expected inputs is missing.
+  step7:
+    subset:
+      - makeCcdVisitTable
+      - makeVisitTable
+      - consolidateHealSparsePropertyMaps
+    description: >
+      Tasks that should be run as the final step that require global inputs,
+      and can be run after the 'step3' subset.
+
+      This step has global aggregation tasks to run over all visits, detectors,
+      tracts, etc.  This step should be run only with the instrument constraint
+      in the data query.

attachments/DRP-ci_hsc+fakes.yaml

@@ -0,0 +1,34 @@
+description: The DRP+fakes pipeline specialized for the ci_hsc test dataset.
+instrument: lsst.obs.subaru.HyperSuprimeCam
+imports:
+  # In the long term, we should try to avoid importing anything from pipelines;
+  # better to refactor anything imported into ingredients.
+  location: "$DRP_PIPE_DIR/pipelines/HSC/DRP-RC2+fakes.yaml"
+  exclude:
+    - faro_step2
+    - faro_step3
+    - faro_step5
+
+tasks:
+  singleFrameWithFakes:
+    class: lsst.pipe.tasks.processCcdWithFakes.ProcessCcdWithFakesTask
+    config:
+      insertFakes.doProcessAllDataIds: true
+      doApplyExternalTractSkyWcs: true
+      doApplyExternalTractPhotoCalib: true
+      doApplyExternalGlobalPhotoCalib: false
+      connections.photoCalibName: "jointcal"
+  makeWarp:
+    class: lsst.pipe.tasks.makeCoaddTempExp.MakeWarpTask
+    config:
+      connections.photoCalibName: "jointcal"
+      useGlobalExternalPhotoCalib: false
+      connections.coaddName: "fakes_deep"
+      connections.calexpType: "fakes_"
+  deblend:
+    class: lsst.pipe.tasks.deblendCoaddSourcesPipeline.DeblendCoaddSourcesMultiTask
+    config:
+      connections.inputCoaddName: "fakes_deep"
+      connections.outputCoaddName: "fakes_deep"
+      multibandDeblend.useCiLimits: true
+      multibandDeblend.processSingles: false