Skip to content

Commit

Permalink
Add gen3 ability to add external SkyWcs and PhotoCalib
Browse files Browse the repository at this point in the history
  • Loading branch information
@erykoff
erykoff committed Mar 1, 2021
1 parent 7fa5663 commit b7b607a
Show file tree
Hide file tree
Showing 2 changed files with 137 additions and 34 deletions.
26 changes: 16 additions & 10 deletions python/lsst/pipe/tasks/coaddBase.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,14 +66,6 @@ class CoaddBaseConfig(pexConfig.Config):
default=False
)
modelPsf = measAlg.GaussianPsfFactory.makeField(doc="Model Psf factory")
doApplyUberCal = pexConfig.Field(
dtype=bool,
doc=("Apply ubercalibrated WCS and PhotoCalib results to input calexps? "
"This field is no longer used, and has been deprecated by DM-21308. "
"It will be removed after v20. Use doApplyExternalPhotoCalib and "
"doApplyExternalSkyWcs instead."),
default=False
)
doApplyExternalPhotoCalib = pexConfig.Field(
dtype=bool,
default=False,
Expand All @@ -82,9 +74,16 @@ class CoaddBaseConfig(pexConfig.Config):
"`externalPhotoCalibName` field to determine which calibration "
"to load.")
)
useGlobalExternalPhotoCalib = pexConfig.Field(
dtype=bool,
default=True,
doc=("When using doApplyExternalPhotoCalib, use global calibrations. "
"When False, per-tract calibrations will be used instead.")
)
externalPhotoCalibName = pexConfig.ChoiceField(
dtype=str,
doc="Type of external PhotoCalib if `doApplyExternalPhotoCalib` is True.",
doc=("Type of external PhotoCalib if `doApplyExternalPhotoCalib` is True. "
"This field is only used for Gen2 middleware."),
default="jointcal",
allowed={
"jointcal": "Use jointcal_photoCalib",
Expand All @@ -99,9 +98,16 @@ class CoaddBaseConfig(pexConfig.Config):
"`lsst.afw.geom.SkyWcs` object. Uses `externalSkyWcsName` "
"field to determine which calibration to load.")
)
useGlobalExternalSkyWcs = pexConfig.Field(
dtype=bool,
default=False,
doc=("When using doApplyExternalSkyWcs, use global calibrations. "
"When False, per-tract calibrations will be used instead.")
)
externalSkyWcsName = pexConfig.ChoiceField(
dtype=str,
doc="Type of external SkyWcs if `doApplyExternalSkyWcs` is True.",
doc=("Type of external SkyWcs if `doApplyExternalSkyWcs` is True. "
"This field is only used for Gen2 middleware."),
default="jointcal",
allowed={
"jointcal": "Use jointcal_wcs"
Expand Down
145 changes: 121 additions & 24 deletions python/lsst/pipe/tasks/makeCoaddTempExp.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
import lsst.afw.image as afwImage
import lsst.coadd.utils as coaddUtils
import lsst.pipe.base as pipeBase
import lsst.pipe.base.connectionTypes as cT
import lsst.pipe.base.connectionTypes as connectionTypes
import lsst.log as log
import lsst.utils as utils
from lsst.meas.algorithms import CoaddPsf, CoaddPsfConfig
Expand Down Expand Up @@ -573,42 +573,68 @@ def applySkyCorr(self, dataRef, calexp):

class MakeWarpConnections(pipeBase.PipelineTaskConnections,
dimensions=("tract", "patch", "skymap", "instrument", "visit"),
defaultTemplates={"coaddName": "deep"}):
calExpList = cT.Input(
defaultTemplates={"coaddName": "deep",
"skyWcsName": "jointcal",
"photoCalibName": "fgcmcal"}):
calExpList = connectionTypes.Input(
doc="Input exposures to be resampled and optionally PSF-matched onto a SkyMap projection/patch",
name="calexp",
storageClass="ExposureF",
dimensions=("instrument", "visit", "detector"),
multiple=True,
)
backgroundList = cT.Input(
backgroundList = connectionTypes.Input(
doc="Input backgrounds to be added back into the calexp if bgSubtracted=False",
name="calexpBackground",
storageClass="Background",
dimensions=("instrument", "visit", "detector"),
multiple=True,
)
skyCorrList = cT.Input(
skyCorrList = connectionTypes.Input(
doc="Input Sky Correction to be subtracted from the calexp if doApplySkyCorr=True",
name="skyCorr",
storageClass="Background",
dimensions=("instrument", "visit", "detector"),
multiple=True,
)
skyMap = cT.Input(
skyMap = connectionTypes.Input(
doc="Input definition of geometry/bbox and projection/wcs for warped exposures",
name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
storageClass="SkyMap",
dimensions=("skymap",),
)
direct = cT.Output(
externalSkyWcsTractCatalog = connectionTypes.Input(
doc="Input external SkyWcs (aggregated per-visit) when run per-tract",
name="{skyWcsName}SkyWcsCatalog",
storageClass="ExposureCatalog",
dimensions=("instrument", "visit", "tract"),
)
externalSkyWcsGlobalCatalog = connectionTypes.Input(
doc="Input external SkyWcs (aggregated per-visit) when run globally",
name="{skyWcsName}SkyWcsCatalog",
storageClass="ExposureCatalog",
dimensions=("instrument", "visit"),
)
externalPhotoCalibTractCatalog = connectionTypes.Input(
doc="Input external PhotoCalib (aggregated per-visit) when run per-tract",
name="{photoCalibName}PhotoCalibCatalog",
storageClass="ExposureCatalog",
dimensions=("instrument", "visit", "tract"),
)
externalPhotoCalibGlobalCatalog = connectionTypes.Input(
doc="Input external PhotoCalib (aggregated per-visit) when run globally",
name="{photoCalibName}PhotoCalibCatalog",
storageClass="ExposureCatalog",
dimensions=("instrument", "visit"),
)
direct = connectionTypes.Output(
doc=("Output direct warped exposure (previously called CoaddTempExp), produced by resampling ",
"calexps onto the skyMap patch geometry."),
name="{coaddName}Coadd_directWarp",
storageClass="ExposureF",
dimensions=("tract", "patch", "skymap", "visit", "instrument"),
)
psfMatched = cT.Output(
psfMatched = connectionTypes.Output(
doc=("Output PSF-Matched warped exposure (previously called CoaddTempExp), produced by resampling ",
"calexps onto the skyMap patch geometry and PSF-matching to a model PSF."),
name="{coaddName}Coadd_psfMatchedWarp",
Expand All @@ -622,6 +648,22 @@ def __init__(self, *, config=None):
self.inputs.remove("backgroundList")
if not config.doApplySkyCorr:
self.inputs.remove("skyCorrList")
if config.doApplyExternalSkyWcs:
if config.useGlobalExternalSkyWcs:
self.inputs.remove("externalSkyWcsTractCatalog")
else:
self.inputs.remove("externalSkyWcsGlobalCatalog")
else:
self.inputs.remove("externalSkyWcsTractCatalog")
self.inputs.remove("externalSkyWcsGlobalCatalog")
if config.doApplyExternalPhotoCalib:
if config.useGlobalExternalPhotoCalib:
self.inputs.remove("externalPhotoCalibTractCatalog")
else:
self.inputs.remove("externalPhotoCalibGlobalCatalog")
else:
self.inputs.remove("externalPhotoCalibTractCatalog")
self.inputs.remove("externalPhotoCalibGlobalCatalog")
if not config.makeDirect:
self.outputs.remove("direct")
if not config.makePsfMatched:
Expand All @@ -633,21 +675,10 @@ class MakeWarpConfig(pipeBase.PipelineTaskConfig, MakeCoaddTempExpConfig,

def validate(self):
super().validate()
# TODO: Remove this constraint after DM-17062
if self.doApplyExternalPhotoCalib:
raise RuntimeError("Gen3 MakeWarpTask cannot apply external PhotoCalib results. "
"Please set doApplyExternalPhotoCalib=False.")
if self.doApplyExternalSkyWcs:
raise RuntimeError("Gen3 MakeWarpTask cannot apply external SkyWcs results. "
"Please set doApplyExternalSkyWcs=False.")


class MakeWarpTask(MakeCoaddTempExpTask, pipeBase.PipelineTask):
"""Warp and optionally PSF-Match calexps onto an a common projection
First Draft of a Gen3 compatible MakeWarpTask which
currently does not handle doApplyExternalPhotoCalib=True or
doApplyExternalSkyWcs=True.
"""
ConfigClass = MakeWarpConfig
_DefaultName = "makeWarp"
Expand Down Expand Up @@ -679,22 +710,37 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs):

# Extract integer visitId requested by `run`
visits = [dataId['visit'] for dataId in dataIdList]
assert(all(visits[0] == visit for visit in visits))
visitId = visits[0]

self.prepareCalibratedExposures(**inputs)
if self.config.doApplyExternalSkyWcs:
if self.config.useGlobalExternalSkyWcs:
externalSkyWcsCatalog = inputs.pop("externalSkyWcsGlobalCatalog")
else:
externalSkyWcsCatalog = inputs.pop("externalSkyWcsTractCatalog")
else:
externalSkyWcsCatalog = None

if self.config.doApplyExternalPhotoCalib:
if self.config.useGlobalExternalPhotoCalib:
externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibGlobalCatalog")
else:
externalPhotoCalibCatalog = inputs.pop("externalPhotoCalibTractCatalog")
else:
externalPhotoCalibCatalog = None

self.prepareCalibratedExposures(**inputs, externalSkyWcsCatalog=externalSkyWcsCatalog,
externalPhotoCalibCatalog=externalPhotoCalibCatalog)
results = self.run(**inputs, visitId=visitId, ccdIdList=ccdIdList, dataIdList=dataIdList,
skyInfo=skyInfo)
if self.config.makeDirect:
butlerQC.put(results.exposures["direct"], outputRefs.direct)
if self.config.makePsfMatched:
butlerQC.put(results.exposures["psfMatched"], outputRefs.psfMatched)

def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None):
def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrList=None,
externalSkyWcsCatalog=None, externalPhotoCalibCatalog=None):
"""Calibrate and add backgrounds to input calExpList in place
TODO DM-17062: apply jointcal/meas_mosaic here
Parameters
----------
calExpList : `list` of `lsst.afw.image.Exposure`
Expand All @@ -703,12 +749,63 @@ def prepareCalibratedExposures(self, calExpList, backgroundList=None, skyCorrLis
Sequence of backgrounds to be added back in if bgSubtracted=False
skyCorrList : `list` of `lsst.afw.math.backgroundList`
Sequence of background corrections to be subtracted if doApplySkyCorr=True
externalSkyWcsCatalog : `lsst.afw.table.ExposureCatalog`
Exposure catalog with external skyWcs to be applied
if config.doApplyExternalSkyWcs=True.
externalPhotoCalibCatalog : `lsst.afw.table.ExposureCatalog`
Exposure catalog with external photoCalib to be applied
if config.doApplyExternalPhotoCalib=True.
"""
backgroundList = len(calExpList)*[None] if backgroundList is None else backgroundList
skyCorrList = len(calExpList)*[None] if skyCorrList is None else skyCorrList

if externalSkyWcsCatalog is not None:
externalSkyWcsDetectors = list(externalSkyWcsCatalog['detector_id'])
if externalPhotoCalibCatalog is not None:
externalPhotoCalibDetectors = list(externalPhotoCalibCatalog['detector_id'])
includeCalibVar = self.config.includeCalibVar

for calexp, background, skyCorr in zip(calExpList, backgroundList, skyCorrList):
mi = calexp.maskedImage
if not self.config.bgSubtracted:
mi += background.getImage()

if externalSkyWcsCatalog is not None or externalPhotoCalibCatalog is not None:
detectorId = calexp.getInfo().getDetector().getId()

# Find and apply external photoCalib
if externalPhotoCalibCatalog is not None:
try:
index = externalPhotoCalibDetectors.index(detectorId)
except ValueError:
raise RuntimeError(f"Detector id {detectorId} not found in "
f"externalPhotoCalibCatalog.")
photoCalib = externalPhotoCalibCatalog[index].getPhotoCalib()
if photoCalib is None:
raise RuntimeError(f"Detector id {detectorId} has None for photoCalib "
f"in externalPhotoCalibCatalog.")
else:
photoCalib = calexp.getPhotoCalib()

# Find and apply external skyWcs
if externalSkyWcsCatalog is not None:
try:
index = externalSkyWcsDetectors.index(detectorId)
except ValueError:
raise RuntimeError(f"Detector id {detectorId} not found in externalSkyWcsCatalog.")
skyWcs = externalSkyWcsCatalog[index].getWcs()
if skyWcs is None:
raise RuntimeError(f"Detector id {detectorId} has None for WCS "
f" in externalSkyWcsCatalog.")
calexp.setWcs(skyWcs)

# Calibrate the image
calexp.maskedImage = photoCalib.calibrateImage(calexp.maskedImage,
includeScaleUncertainty=includeCalibVar)
calexp.maskedImage /= photoCalib.getCalibrationMean()
# TODO: The images will have a calibration of 1.0 everywhere once RFC-545 is implemented.
# exposure.setCalib(afwImage.Calib(1.0))

# Apply skycorr
if self.config.doApplySkyCorr:
mi -= skyCorr.getImage()

0 comments on commit b7b607a

Please sign in to comment.