Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

DM-31491: Add default types to allow fakes propagation #576

Merged
merged 2 commits into from
Oct 27, 2021
Merged

DM-31491: Add default types to allow fakes propagation #576

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cb1725a
Add default types to allow fakes propagation
sr525 Sep 23, 2021
677a96b
Update config options for use with fgcm and jointcal
sr525 Oct 20, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
5 changes: 3 additions & 2 deletions python/lsst/pipe/tasks/healSparseMapping.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,7 +311,8 @@ def _pixels_to_radec(self, wcs, pixels):

class HealSparsePropertyMapConnections(pipeBase.PipelineTaskConnections,
dimensions=("tract", "band", "skymap",),
defaultTemplates={"coaddName": "deep"}):
defaultTemplates={"coaddName": "deep",
"calexpType": ""}):
input_maps = pipeBase.connectionTypes.Input(
doc="Healsparse bit-wise coadd input maps",
name="{coaddName}Coadd_inputMap",
Expand All @@ -330,7 +331,7 @@ class HealSparsePropertyMapConnections(pipeBase.PipelineTaskConnections,
)
visit_summaries = pipeBase.connectionTypes.Input(
doc="Visit summary tables with aggregated statistics",
name="visitSummary",
name="{calexpType}visitSummary",
storageClass="ExposureCatalog",
dimensions=("instrument", "visit"),
multiple=True,
Expand Down
11 changes: 11 additions & 0 deletions python/lsst/pipe/tasks/insertFakes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ def _add_fake_sources(exposure, objects, calibFluxRadius=12.0, logger=None):
fullBounds = galsim.BoundsI(bbox.minX, bbox.maxX, bbox.minY, bbox.maxY)
gsImg = galsim.Image(exposure.image.array, bounds=fullBounds)

pixScale = wcs.getPixelScale().asArcseconds()

for spt, gsObj in objects:
pt = wcs.skyToPixel(spt)
posd = galsim.PositionD(pt.x, pt.y)
Expand All @@ -81,6 +83,13 @@ def _add_fake_sources(exposure, objects, calibFluxRadius=12.0, logger=None):
mat = wcs.linearizePixelToSky(spt, geom.arcseconds).getMatrix()
gsWCS = galsim.JacobianWCS(mat[0, 0], mat[0, 1], mat[1, 0], mat[1, 1])

# This check is here because sometimes the WCS
# is multivalued and objects that should not be
# were being included.
gsPixScale = np.sqrt(gsWCS.pixelArea())
if gsPixScale < pixScale/2 or gsPixScale > pixScale*2:
Comment on lines +86 to +90
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is a gsPixScale offset of a factor of 2 sufficient to always capture this multivalued WCS issue, or will you still expect some level of failure? If sufficient, then that's fine. If it might continue to be an issue however, then perhaps this factor should be configurable?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was sufficient in testing, I think if it turns out not to be we need to do something more intelligent than this check. In which case I would like it to crash so we can figure out a better solution.

continue

try:
psfArr = psf.computeKernelImage(pt).array
except InvalidParameterError:
Expand All @@ -106,6 +115,7 @@ def _add_fake_sources(exposure, objects, calibFluxRadius=12.0, logger=None):
pt
)
continue

apCorr = psf.computeApertureFlux(calibFluxRadius)
psfArr /= apCorr
gsPSF = galsim.InterpolatedImage(galsim.Image(psfArr), wcs=gsWCS)
Expand All @@ -121,6 +131,7 @@ def _add_fake_sources(exposure, objects, calibFluxRadius=12.0, logger=None):
# Note, for calexp injection, pixel is already part of the PSF and
# for coadd injection, it's incorrect to include the output pixel.
# So for both cases, we draw using method='no_pixel'.

conv.drawImage(
subImg,
add_to_image=True,
Expand Down
8 changes: 4 additions & 4 deletions python/lsst/pipe/tasks/postprocess.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1394,10 +1394,10 @@ def writeConfig(self, butler, clobber=False, doBackup=True):

class MakeCcdVisitTableConnections(pipeBase.PipelineTaskConnections,
dimensions=("instrument",),
defaultTemplates={}):
defaultTemplates={"calexpType": ""}):
visitSummaryRefs = connectionTypes.Input(
doc="Data references for per-visit consolidated exposure metadata from ConsolidateVisitSummaryTask",
name="visitSummary",
name="{calexpType}visitSummary",
storageClass="ExposureCatalog",
dimensions=("instrument", "visit"),
multiple=True,
Expand Down Expand Up @@ -1486,10 +1486,10 @@ def run(self, visitSummaryRefs):

class MakeVisitTableConnections(pipeBase.PipelineTaskConnections,
dimensions=("instrument",),
defaultTemplates={}):
defaultTemplates={"calexpType": ""}):
visitSummaries = connectionTypes.Input(
doc="Per-visit consolidated exposure metadata from ConsolidateVisitSummaryTask",
name="visitSummary",
name="{calexpType}visitSummary",
storageClass="ExposureCatalog",
dimensions=("instrument", "visit",),
multiple=True,
Expand Down
108 changes: 96 additions & 12 deletions python/lsst/pipe/tasks/processCcdWithFakes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,11 +70,44 @@ class ProcessCcdWithFakesConnections(PipelineTaskConnections,
dimensions=("tract", "skymap", "instrument", "visit", "detector")
)

externalSkyWcsTractCatalog = cT.Input(
doc=("Per-tract, per-visit wcs calibrations. These catalogs use the detector "
"id for the catalog id, sorted on id for fast lookup."),
name="{wcsName}SkyWcsCatalog",
storageClass="ExposureCatalog",
dimensions=("instrument", "visit", "tract", "skymap"),
)

externalSkyWcsGlobalCatalog = cT.Input(
doc=("Per-visit wcs calibrations computed globally (with no tract information). "
"These catalogs use the detector id for the catalog id, sorted on is for "
"fast lookup."),
name="{wcsName}SkyWcsCatalog",
storageClass="ExposureCatalog",
dimensions=("instrument", "visit"),
)

photoCalib = cT.Input(
doc="Calib information for the input exposure.",
name="{photoCalibName}_photoCalib",
storageClass="PhotoCalib",
dimensions=("tract", "skymap", "instrument", "visit", "detector")
dimensions=("instrument", "visit", "detector", "tract", "skymap")
)

externalPhotoCalibTractCatalog = cT.Input(
doc=("Per-tract, per-visit photometric calibrations. These catalogs use the "
"detector id for the catalog id, sorted on id for fast lookup."),
name="{photoCalibName}PhotoCalibCatalog",
storageClass="ExposureCatalog",
dimensions=("instrument", "visit", "tract"),
)

externalPhotoCalibGlobalCatalog = cT.Input(
doc=("Per-visit photometric calibrations. These catalogs use the "
"detector id for the catalog id, sorted on id for fast lookup."),
name="{photoCalibName}PhotoCalibCatalog",
storageClass="ExposureCatalog",
dimensions=("instrument", "visit"),
)

icSourceCat = cT.Input(
Expand Down Expand Up @@ -108,11 +141,21 @@ class ProcessCcdWithFakesConnections(PipelineTaskConnections,
def __init__(self, *, config=None):
super().__init__(config=config)

if config.doApplyExternalSkyWcs is False:
if config.doApplyExternalGlobalSkyWcs or config.doApplyExternalTractSkyWcs:
self.inputs.remove("wcs")
if config.doApplyExternalPhotoCalib is False:
if config.doApplyExternalGlobalPhotoCalib or config.doApplyExternalTractPhotoCalib:
self.inputs.remove("photoCalib")

if not config.doApplyExternalGlobalPhotoCalib:
self.inputs.remove("externalPhotoCalibGlobalCatalog")
if not config.doApplyExternalTractPhotoCalib:
self.inputs.remove("externalPhotoCalibTractCatalog")

if not config.doApplyExternalGlobalSkyWcs:
self.inputs.remove("externalSkyWcsGlobalCatalog")
if not config.doApplyExternalTractSkyWcs:
self.inputs.remove("externalSkyWcsTractCatalog")


class ProcessCcdWithFakesConfig(PipelineTaskConfig,
pipelineConnections=ProcessCcdWithFakesConnections):
Expand All @@ -123,13 +166,22 @@ class ProcessCcdWithFakesConfig(PipelineTaskConfig,
The default column names are those from the UW sims database.
"""

doApplyExternalPhotoCalib = pexConfig.Field(
doApplyExternalGlobalPhotoCalib = pexConfig.Field(
dtype=bool,
default=False,
doc="Whether to apply an external photometric calibration via an "
"`lsst.afw.image.PhotoCalib` object. Uses the "
"`externalPhotoCalibName` config option to determine which "
"calibration to use."
"calibration to use. Uses a global calibration."
)

doApplyExternalTractPhotoCalib = pexConfig.Field(
dtype=bool,
default=False,
doc="Whether to apply an external photometric calibration via an "
"`lsst.afw.image.PhotoCalib` object. Uses the "
"`externalPhotoCalibName` config option to determine which "
"calibration to use. Uses a per tract calibration."
)

externalPhotoCalibName = pexConfig.ChoiceField(
Expand All @@ -141,13 +193,22 @@ class ProcessCcdWithFakesConfig(PipelineTaskConfig,
"fgcm_tract": "Use fgcm_tract_photoCalib"}
)

doApplyExternalSkyWcs = pexConfig.Field(
doApplyExternalGlobalSkyWcs = pexConfig.Field(
dtype=bool,
default=False,
doc="Whether to apply an external astrometric calibration via an "
"`lsst.afw.geom.SkyWcs` object. Uses the "
"`externalSkyWcsName` config option to determine which "
"calibration to use."
"calibration to use. Uses a global calibration."
)

doApplyExternalTractSkyWcs = pexConfig.Field(
dtype=bool,
default=False,
doc="Whether to apply an external astrometric calibration via an "
"`lsst.afw.geom.SkyWcs` object. Uses the "
"`externalSkyWcsName` config option to determine which "
"calibration to use. Uses a per tract calibration."
)

externalSkyWcsName = pexConfig.ChoiceField(
Expand Down Expand Up @@ -270,13 +331,13 @@ def runDataRef(self, dataRef):
fakeCat = Table.read(self.config.insertFakes.fakeType).to_pandas()

calexp = dataRef.get("calexp")
if self.config.doApplyExternalSkyWcs:
if self.config.doApplyExternalGlobalSkyWcs or self.config.doApplyExternalTractSkyWcs:
self.log.info("Using external wcs from %s", self.config.externalSkyWcsName)
wcs = dataRef.get(self.config.externalSkyWcsName + "_wcs")
else:
wcs = calexp.getWcs()

if self.config.doApplyExternalPhotoCalib:
if self.config.doApplyExternalGlobalPhotoCalib or self.config.doApplyExternalTractPhotoCalib:
self.log.info("Using external photocalib from %s", self.config.externalPhotoCalibName)
photoCalib = dataRef.get(self.config.externalPhotoCalibName + "_photoCalib")
else:
Expand All @@ -295,16 +356,38 @@ def runDataRef(self, dataRef):

def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputs = butlerQC.get(inputRefs)
detectorId = inputs["exposure"].getInfo().getDetector().getId()

if 'exposureIdInfo' not in inputs.keys():
expId, expBits = butlerQC.quantum.dataId.pack("visit_detector", returnMaxBits=True)
inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)

if not self.config.doApplyExternalSkyWcs:
if not self.config.doApplyExternalGlobalSkyWcs and not self.config.doApplyExternalTractSkyWcs:
inputs["wcs"] = inputs["exposure"].getWcs()

if not self.config.doApplyExternalPhotoCalib:
elif self.config.doApplyExternalGlobalSkyWcs:
externalSkyWcsCatalog = inputs["externalSkyWcsGobalCatalog"]
row = externalSkyWcsCatalog.find(detectorId)
inputs["wcs"] = row.getWcs()

elif self.config.doApplyExternalTractSkyWcs:
externalSkyWcsCatalog = inputs["externalSkyWcsTractCatalog"]
row = externalSkyWcsCatalog.find(detectorId)
inputs["wcs"] = row.getWcs()

if not self.config.doApplyExternalGlobalPhotoCalib and not self.config.doApplyExternalTractPhotoCalib:
inputs["photoCalib"] = inputs["exposure"].getPhotoCalib()

elif self.config.doApplyExternalGlobalPhotoCalib:
externalPhotoCalibCatalog = inputs["externalPhotoCalibGlobalCatalog"]
row = externalPhotoCalibCatalog.find(detectorId)
inputs["photoCalib"] = row.getPhotoCalib()

elif self.config.doApplyExternalTractPhotoCalib:
externalPhotoCalibCatalog = inputs["externalPhotoCalibTractCatalog"]
row = externalPhotoCalibCatalog.find(detectorId)
inputs["photoCalib"] = row.getPhotoCalib()

outputs = self.run(**inputs)
butlerQC.put(outputs, outputRefs)

Expand All @@ -317,7 +400,8 @@ def _makeArgumentParser(cls):
return parser

def run(self, fakeCat, exposure, wcs=None, photoCalib=None, exposureIdInfo=None, icSourceCat=None,
sfdSourceCat=None):
sfdSourceCat=None, externalSkyWcsGlobalCatalog=None, externalSkyWcsTractCatalog=None,
externalPhotoCalibGlobalCatalog=None, externalPhotoCalibTractCatalog=None):
"""Add fake sources to a calexp and then run detection, deblending and measurement.

Parameters
Expand Down