Create DECam DRP Processing Pipeline

config/apertures.py

@@ -0,0 +1,5 @@
+# Set up aperture photometry
+# 'config' should be a SourceMeasurementConfig
+
+# Use a large aperture to be independent of seeing in calibration
+config.plugins["base_CircularApertureFlux"].maxSincRadius = 12.0

config/assembleCoadd.py

@@ -0,0 +1,18 @@
+import os.path
+
+# Load configs shared between makeWarp and assembleCoadd
+config.load(os.path.join(os.path.dirname(__file__), "coaddBase.py"))
+
+# 200 rows (since patch width is typically < 10k pixels)
+config.subregionSize = (10000, 200)
+config.doMaskBrightObjects = True
+config.removeMaskPlanes.append("CROSSTALK")
+config.doNImage = True
+config.badMaskPlanes += ["SUSPECT"]
+config.doAttachTransmissionCurve = True
+# Saturation trails are usually oriented east-west, so along rows
+config.interpImage.transpose = True
+config.coaddPsf.warpingKernelName = 'lanczos5'
+
+from lsst.pipe.tasks.selectImages import PsfWcsSelectImagesTask
+config.select.retarget(PsfWcsSelectImagesTask)

config/calibrate.py

@@ -3,8 +3,12 @@
 """
 import os.path
 
+from lsst.meas.algorithms import ColorLimit
+from lsst.meas.astrom import MatchOptimisticBConfig
+
 obsConfigDir = os.path.join(os.path.dirname(__file__))
 
+# Astrometry/Photometry
 # This sets the reference catalog name for Gen2.
 for refObjLoader in (config.astromRefObjLoader,
                      config.photoRefObjLoader,
@@ -21,7 +25,21 @@
 # This sets up the reference catalog for Gen3.
 config.connections.astromRefCat = "ps1_pv3_3pi_20170110"
 config.connections.photoRefCat = "ps1_pv3_3pi_20170110"
+
+# Photometric calibration: use color terms
+config.photoCal.applyColorTerms = True
 config.photoCal.photoCatName = "ps1_pv3_3pi_20170110"
+colors = config.photoCal.match.referenceSelection.colorLimits
+# The following two color limits are adopted from obs_subaru for the HSC SSP survey
+colors["g-r"] = ColorLimit(primary="g_flux", secondary="r_flux", minimum=0.0)
+colors["r-i"] = ColorLimit(primary="r_flux", secondary="i_flux", maximum=0.5)
+config.photoCal.match.referenceSelection.doMagLimit = True
+config.photoCal.match.referenceSelection.magLimit.fluxField = "i_flux"
+config.photoCal.match.referenceSelection.magLimit.maximum = 22.0
+config.photoCal.colorterms.load(os.path.join(obsConfigDir, 'colorterms.py'))
+
+# Better astrometry matching
+config.astrometry.matcher.numBrightStars = 150
 
 # The Task default was reduced from 4 to 2 on RFC-577. We believe that 4 is
 # more appropriate for use with DECam data until a Jointcal-derived distortion
@@ -30,4 +48,37 @@
 # See Slack: https://lsstc.slack.com/archives/C2B6X08LS/p1586468459084600
 config.astrometry.wcsFitter.order = 4
 
+for matchConfig in (config.astrometry,
+                    ):
+    matchConfig.sourceFluxType = 'Psf'
+    matchConfig.sourceSelector.active.sourceFluxType = 'Psf'
+    matchConfig.matcher.maxRotationDeg = 1.145916
+    matchConfig.matcher.maxOffsetPix = 250
+    if isinstance(matchConfig.matcher, MatchOptimisticBConfig):
+        matchConfig.matcher.allowedNonperpDeg = 0.2
+        matchConfig.matcher.maxMatchDistArcSec = 2.0
+        matchConfig.sourceSelector.active.excludePixelFlags = False
+
+# Set to match defaults curretnly used in HSC production runs (e.g. S15B)
+config.catalogCalculation.plugins['base_ClassificationExtendedness'].fluxRatio = 0.95
+
+# Demand astrometry and photoCal succeed
+config.requireAstrometry = True
+config.requirePhotoCal = True
+
+config.doWriteMatchesDenormalized = True
+
+# Detection
+config.detection.isotropicGrow = True
+
+config.measurement.load(os.path.join(obsConfigDir, "apertures.py"))
+config.measurement.load(os.path.join(obsConfigDir, "kron.py"))
 config.measurement.load(os.path.join(obsConfigDir, "hsm.py"))
+
+# Deblender
+config.deblend.maxFootprintSize = 0
+config.deblend.maskLimits["NO_DATA"] = 0.25  # Ignore sources that are in the vignetted region
+config.deblend.maxFootprintArea = 10000
+
+config.measurement.plugins.names |= ["base_Jacobian", "base_FPPosition"]
+config.measurement.plugins["base_Jacobian"].pixelScale = 0.263

config/characterizeImage.py

@@ -1,8 +1,23 @@
 """
 DECam-specific overrides for CharacterizeImageTask
 """
+import os.path
+
+from lsst.meas.astrom import MatchOptimisticBConfig
+
+obsConfigDir = os.path.dirname(__file__)
+
+# Cosmic rays
 config.repair.cosmicray.nCrPixelMax = 100000
+config.repair.cosmicray.cond3_fac2 = 0.4
+
+# PSF determination
+config.measurePsf.reserve.fraction = 0.2
+config.measurePsf.starSelector["objectSize"].sourceFluxField = "base_PsfFlux_instFlux"
+config.measurePsf.starSelector["objectSize"].widthMin = 0.9
+config.measurePsf.starSelector["objectSize"].fluxMin = 4000
 
+# Astrometry/Photometry
 # This sets the reference catalog name for Gen2.
 # Note that in Gen3, we've stopped pretending (which is what Gen2 does,
 # for backwards compatibility) that charImage uses a reference catalog.
@@ -14,3 +29,61 @@
 config.refObjLoader.filterMap['N540'] = 'g'
 config.refObjLoader.filterMap['N708'] = 'i'
 config.refObjLoader.filterMap['N964'] = 'z'
+
+# Set to match defaults currently used in HSC production runs (e.g. S15B)
+config.catalogCalculation.plugins['base_ClassificationExtendedness'].fluxRatio = 0.95
+
+# Detection
+config.detection.isotropicGrow = True
+
+# Activate calibration of measurements: required for aperture corrections
+config.load(os.path.join(obsConfigDir, "cmodel.py"))
+config.measurement.load(os.path.join(obsConfigDir, "apertures.py"))
+config.measurement.load(os.path.join(obsConfigDir, "kron.py"))
+config.measurement.load(os.path.join(obsConfigDir, "convolvedFluxes.py"))
+config.measurement.load(os.path.join(obsConfigDir, "gaap.py"))
+config.measurement.load(os.path.join(obsConfigDir, "hsm.py"))
+if "ext_shapeHSM_HsmShapeRegauss" in config.measurement.plugins:
+    # no deblending has been done
+    config.measurement.plugins["ext_shapeHSM_HsmShapeRegauss"].deblendNChild = ""
+
+# Deblender
+config.deblend.maskLimits["NO_DATA"] = 0.25  # Ignore sources that are in the vignetted region
+config.deblend.maxFootprintArea = 10000
+
+config.measurement.plugins.names |= ["base_Jacobian", "base_FPPosition"]
+config.measurement.plugins["base_Jacobian"].pixelScale = 0.263
+
+# Convolved fluxes can fail for small target seeing if the observation seeing is larger
+if "ext_convolved_ConvolvedFlux" in config.measurement.plugins:
+    names = config.measurement.plugins["ext_convolved_ConvolvedFlux"].getAllResultNames()
+    config.measureApCorr.allowFailure += names
+
+if "ext_gaap_GaapFlux" in config.measurement.plugins:
+    names = config.measurement.plugins["ext_gaap_GaapFlux"].getAllGaapResultNames()
+    config.measureApCorr.allowFailure += names
+
+# For aperture correction modeling, only use objects that were used in the
+# PSF model and have psf flux signal-to-noise > 200.
+config.measureApCorr.sourceSelector['science'].doFlags = True
+config.measureApCorr.sourceSelector['science'].doUnresolved = False
+config.measureApCorr.sourceSelector['science'].doSignalToNoise = True
+config.measureApCorr.sourceSelector['science'].flags.good = ["calib_psf_used"]
+config.measureApCorr.sourceSelector['science'].flags.bad = []
+config.measureApCorr.sourceSelector['science'].signalToNoise.minimum = 200.0
+config.measureApCorr.sourceSelector['science'].signalToNoise.maximum = None
+config.measureApCorr.sourceSelector['science'].signalToNoise.fluxField = "base_PsfFlux_instFlux"
+config.measureApCorr.sourceSelector['science'].signalToNoise.errField = "base_PsfFlux_instFluxErr"
+config.measureApCorr.sourceSelector.name = "science"
+
+config.ref_match.sourceSelector.name = 'matcher'
+for matchConfig in (config.ref_match,
+                    ):
+    matchConfig.sourceFluxType = 'Psf'
+    matchConfig.sourceSelector.active.sourceFluxType = 'Psf'
+    matchConfig.matcher.maxRotationDeg = 1.145916
+    matchConfig.matcher.maxOffsetPix = 250
+    if isinstance(matchConfig.matcher, MatchOptimisticBConfig):
+        matchConfig.matcher.allowedNonperpDeg = 0.2
+        matchConfig.matcher.maxMatchDistArcSec = 2.0
+        matchConfig.sourceSelector.active.excludePixelFlags = False

config/cmodel.py

@@ -0,0 +1,9 @@
+# Enable CModel mags (unsetup meas_modelfit to disable)
+# 'config' is a SourceMeasurementConfig.
+try:
+    import lsst.meas.modelfit
+    config.measurement.plugins.names |= ["modelfit_DoubleShapeletPsfApprox", "modelfit_CModel"]
+    config.measurement.slots.modelFlux = 'modelfit_CModel'
+    config.catalogCalculation.plugins['base_ClassificationExtendedness'].fluxRatio = 0.985
+except (KeyError, ImportError):
+    print("Cannot import lsst.meas.modelfit: disabling CModel measurements")

config/coaddBase.py

@@ -0,0 +1,9 @@
+# Configs shared between makeWarp and assembleCoadd
+config.matchingKernelSize = 29
+config.doApplyExternalPhotoCalib = True
+config.externalPhotoCalibName = 'jointcal'
+config.doApplyExternalSkyWcs = True
+
+# When useGlobalExternalPhotoCalib is set to False, use per-tract photometric calibrations.
+# Global calibrations must be disabled if using jointcal.
+config.useGlobalExternalPhotoCalib = False

config/colorAnalysis.py

@@ -0,0 +1,5 @@
+import os.path
+
+config.load(os.path.join(os.path.dirname(__file__), "extinctionCoeffs.py"))
+from lsst.obs.decam.decamFilters import DECAM_FILTER_DEFINITIONS
+config.physicalToBandFilterMap = DECAM_FILTER_DEFINITIONS.physical_to_band

config/colorterms.py

@@ -0,0 +1,52 @@
+from lsst.pipe.tasks.colorterms import Colorterm, ColortermDict
+
+# Average color terms for the DECam filters are calculated using the Pickles stellar spectra atlas.
+# The following values are provided by Song Huang (Tsinghua University; dr.guangtou@gmail.com)
+# Color terms for u, Y, N419 and N964 bands are not currently available.
+
+# The values for DECam g, r & z-bands are calculated independently and are different to the values at:
+# https://www.legacysurvey.org/dr9/description/
+
+# A Jupyter notebook to reproduce these values is available at:
+# https://github.com/MerianSurvey/caterpillar/blob/main/notebook/photocal/merian_filter_color_terms.ipynb 
+
+# In obs_subaru, colorterms.py is also used in:
+# - config/skyAnalysis.py
+# - config/compareCoaddAnalysis.py 
+# - config/coaddAnalysis.py
+# - config/visitAnalysis.py
+# - config/fgcmBuildStarsTable.py and config/fgcmBuildStars.py
+# These files are not currently available for obs_decam.
+
+config.data = {
+    "decam*": ColortermDict(data={
+        'g DECam SDSS c0001 4720.0 1520.0': Colorterm(primary="g", secondary="g"),
+        'r DECam SDSS c0002 6415.0 1480.0': Colorterm(primary="r", secondary="r"),
+        'i DECam SDSS c0003 7835.0 1470.0': Colorterm(primary="i", secondary="i"),
+        'z DECam SDSS c0004 9260.0 1520.0': Colorterm(primary="z", secondary="z"),
+    }),
+    "sdss*": ColortermDict(data={
+        'g DECam SDSS c0001 4720.0 1520.0': Colorterm(primary="g", secondary="r", c0=-0.008015, c1=-0.089869, c2=-0.018398), # For -1.0 < g-r < 1.8
+        'r DECam SDSS c0002 6415.0 1480.0': Colorterm(primary="r", secondary="i", c0=-0.0077434, c1=-0.202615, c2=0.016042), # For -1.0 < r-i < 2.2
+        'i DECam SDSS c0003 7835.0 1470.0': Colorterm(primary="i", secondary="z", c0=0.00018887, c1=-0.2748281, c2=-0.029417), # For -1.0 < i-z < 1.2
+        'z DECam SDSS c0004 9260.0 1520.0': Colorterm(primary="z", secondary="i", c0=-0.0059858, c1=0.1131192, c2=0.0156471), # For -1.0 < z-i < 0.5
+        'N540 DECam c0014 5403.2 210.0': Colorterm(primary="g", secondary="r", c0=-0.03359776, c1=-0.57665033, c2=-0.01510509), # For -1.0 < g-r < 1.5
+        'N708 DECam c0012 7080.0 400.0': Colorterm(primary="r", secondary="i", c0=-0.01428724, c1=-0.71755905, c2=0.08959316), # For -1.0 < r-i < 2.0
+    }),
+    "hsc*": ColortermDict(data={
+        'g DECam SDSS c0001 4720.0 1520.0': Colorterm(primary="g", secondary="r", c0=0.0000509, c1=-0.0152487, c2=-0.0029937), # For -1.0 < g-r < 1.4
+        'r DECam SDSS c0002 6415.0 1480.0': Colorterm(primary="r", secondary="i", c0=-0.0078995, c1=-0.1695323, c2=0.0251978), # For -1.0 < r-i < 2.2
+        'i DECam SDSS c0003 7835.0 1470.0': Colorterm(primary="i", secondary="z", c0=0.0010196, c1=-0.1314031, c2=0.0054605), # For -1.0 < i-z < 1.0
+        'z DECam SDSS c0004 9260.0 1520.0': Colorterm(primary="z", secondary="y", c0=-0.0013244, c1=-0.2846684, c2=-0.1229817), # For -1.0 < z-y < 0.5
+        'N540 DECam c0014 5403.2 210.0': Colorterm(primary="g", secondary="r", c0=-0.03015158, c1=-0.53715834, c2=-0.00202415), # For -1.0 < g-r < 1.5
+        'N708 DECam c0012 7080.0 400.0': Colorterm(primary="r", secondary="i", c0=-0.01978413, c1=-0.64368838, c2=0.09132738), # For -1.0 < r-i < 2.0
+    }),
+    "ps1*": ColortermDict(data={
+        'g DECam SDSS c0001 4720.0 1520.0': Colorterm(primary="g", secondary="r", c0=0.006388, c1=0.045875, c2=-0.004484), # For -1.0 < g-r < 1.2
+        'r DECam SDSS c0002 6415.0 1480.0': Colorterm(primary="r", secondary="i", c0=-0.006775, c1=-0.187304, c2=0.018928), # For -1.0 < r-i < 2.2
+        'i DECam SDSS c0003 7835.0 1470.0': Colorterm(primary="i", secondary="z", c0=0.0012204, c1=-0.282956, c2=-0.011321), # For -1.0 < i-z < 1.4
+        'z DECam SDSS c0004 9260.0 1520.0': Colorterm(primary="z", secondary="y", c0=-0.0087868, c1=-0.5204795, c2=-0.057955), # For -1.0 < z-y < 1.0
+        'N540 DECam c0014 5403.2 210.0': Colorterm(primary="g", secondary="r", c0=-0.02635164, c1=-0.50968428, c2=-0.00958104), # For -1.0 < g-r < 1.5
+        'N708 DECam c0012 7080.0 400.0': Colorterm(primary="r", secondary="i", c0=-0.01508987, c1=-0.69377675, c2=0.09079348), # For -1.0 < r-i < 2.0
+    }),
+}

config/compareWarpAssembleCoadd.py

@@ -0,0 +1,15 @@
+import os.path
+
+# Load configs from base assembleCoadd
+config.load(os.path.join(os.path.dirname(__file__), "assembleCoadd.py"))
+
+# 200 rows (since patch width is typically < 10k pixels
+config.assembleStaticSkyModel.subregionSize = (10000, 200)
+config.assembleStaticSkyModel.doApplyExternalPhotoCalib = True
+config.assembleStaticSkyModel.externalPhotoCalibName = 'jointcal'
+config.assembleStaticSkyModel.doApplyExternalSkyWcs = True
+config.doFilterMorphological = True
+
+# When useGlobalExternalPhotoCalib is set to False, use per-tract photometric calibrations.
+# Global calibrations must be disabled if using jointcal.
+config.assembleStaticSkyModel.useGlobalExternalPhotoCalib = False

config/convolvedFluxes.py

@@ -0,0 +1,9 @@
+# Enable measurement of convolved fluxes
+# 'config' is a SourceMeasurementConfig
+try:
+    import lsst.meas.extensions.convolved  # noqa: Load flux.convolved algorithm
+except ImportError as exc:
+    print("Cannot import lsst.meas.extensions.convolved (%s): disabling convolved flux measurements" % (exc,))
+else:
+    config.plugins.names.add("ext_convolved_ConvolvedFlux")
+    config.plugins["ext_convolved_ConvolvedFlux"].seeing.append(8.0)

config/extinctionCoeffs.py

@@ -0,0 +1,15 @@
+# Extinction coefficients for DECam filters for conversion from E(B-V) to extinction, A_filter.
+# Numbers initially provided by Song Huang (NAOJ).
+#
+# Band, A_filter/E(B-V)
+# Values for N419 and N964 filters are not currently available.
+config.extinctionCoeffs = {
+    "u": 3.994,
+    "g": 3.212,
+    "r": 2.164,
+    "i": 1.591,
+    "z": 1.211,
+    "y": 1.063,
+    "N540": 2.753,
+    "N708": 1.847,
+}

config/gaap.py

@@ -0,0 +1,10 @@
+# Enable GAaP (Gaussian Aperture and PSF) colors
+# 'config' is typically a SourceMeasurementConfig
+try:
+    import lsst.meas.extensions.gaap  # noqa
+    config.plugins.names.add("ext_gaap_GaapFlux")
+    config.plugins["ext_gaap_GaapFlux"].sigmas = [0.5, 0.7, 1.0, 1.5, 2.5, 3.0]
+    # Enable PSF photometry after PSF-Gaussianization in the `ext_gaap_GaapFlux` plugin
+    config.plugins["ext_gaap_GaapFlux"].doPsfPhotometry = True
+except ImportError as exc:
+    print("Cannot import lsst.meas.extensions.gaap (%s): disabling GAaP flux measurements" % (exc,))

config/jointcal.py

@@ -0,0 +1,20 @@
+import os.path
+
+# `load()` appends to the filterMaps: we need them to be empty for HSC, so that
+# only the specified filter mappings are used.
+# config.photometryRefObjLoader.filterMap = {}
+# filterMapFile = os.path.join(os.path.dirname(__file__), "filterMap.py")
+# config.photometryRefObjLoader.load(filterMapFile)
+# We have PS1 colorterms for HSC.
+config.applyColorTerms = True
+config.colorterms.load(os.path.join(os.path.dirname(__file__), "colorterms.py"))
+
+# HSC needs a higher order polynomial to track the steepness of the optical
+# distortions along the edge of the field. Emperically, this provides a
+# measurably better fit than the default order=5.
+config.astrometryVisitOrder = 7
+
+# For the deep fields with hundreds of visits, outlier rejection may take many
+# weeks of runtime. By stopping outlier rejection when it ceases to have a
+# significant effect on the model, we can bring compute time down to a few days.
+config.astrometryOutlierRelativeTolerance = 0.002

config/kron.py

@@ -0,0 +1,8 @@
+# Enable Kron mags
+# 'config' is a SourceMeasurementConfig
+
+try:
+    import lsst.meas.extensions.photometryKron
+    config.plugins.names |= ["ext_photometryKron_KronFlux"]
+except ImportError:
+    print("Cannot import lsst.meas.extensions.photometryKron: disabling Kron measurements")

config/makeWarp.py

@@ -0,0 +1,11 @@
+import os.path
+
+# Load configs shared between makeWarp and assembleCoadd
+config.load(os.path.join(os.path.dirname(__file__), "coaddBase.py"))
+
+config.doApplySkyCorr = False
+
+config.modelPsf.defaultFwhm = 7.7
+config.warpAndPsfMatch.psfMatch.kernel['AL'].alardSigGauss = [1.0, 2.0, 4.5]
+config.warpAndPsfMatch.warp.warpingKernelName = 'lanczos5'
+config.coaddPsf.warpingKernelName = 'lanczos5'

config/transformSourceTable.py

@@ -0,0 +1,5 @@
+import os.path
+
+# Use the environment variable to prevent hardcoding of paths
+# into quantum graphs.
+config.functorFile = os.path.join('$OBS_DECAM_DIR', 'policy', 'Source.yaml')