Skip to content

Commit

Permalink
Post-review clean-up.
Browse files Browse the repository at this point in the history 
External wcs is now doApplyExternalSkyWcs and externalSkyWcsName.
  • Loading branch information
@erykoff
erykoff committed Dec 6, 2019
1 parent 3e18709 commit cae20c0
Show file tree
Hide file tree
Showing 3 changed files with 181 additions and 101 deletions.
51 changes: 31 additions & 20 deletions python/lsst/validate/drp/matchedVisitMetricsTask.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,9 @@
import os
import sys
import traceback

import lsst.pipe.base as pipeBase
from lsst.pipe.base import CmdLineTask, ArgumentParser, TaskRunner
from lsst.pex.config import Config, Field
from lsst.pex.config import Config, Field, ChoiceField
from lsst.meas.base.forcedPhotCcd import PerTractCcdDataIdContainer
from .validate import runOneFilter, plot_metrics

Expand Down Expand Up @@ -87,22 +86,34 @@ class MatchedVisitMetricsConfig(Config):
)
doApplyExternalPhotoCalib = Field(
dtype=bool, default=False,
doc="Whether to apply external photometric calibration (PhotoCalib)"
doc=("Whether to apply external photometric calibration via an "
"`lsst.afw.image.PhotoCalib` object. Uses the "
"`externalPhotoCalibName` field to determine which calibration "
"to load.")
)
externalPhotoCalibName = Field(
externalPhotoCalibName = ChoiceField(
dtype=str,
doc=("Type of external PhotoCalib. Currently supported are jointcal, "
"fgcm, and fgcm_tract."),
default="jointcal"
doc="Type of external PhotoCalib if `doApplyExternalPhotoCalib` is True.",
default="jointcal",
allowed={
"jointcal": "Use jointcal_photoCalib",
"fgcm": "Use fgcm_photoCalib",
"fgcm_tract": "Use fgcm_tract_photoCalib"
}
)
doApplyExternalWcs = Field(
doApplyExternalSkyWcs = Field(
dtype=bool, default=False,
doc="Whether to apply external astrometric calibration (wcs)"
doc=("Whether to apply external astrometric calibration via an "
"`lsst.afw.geom.SkyWcs` object. Uses the `externalSkyWcsName` "
"field to determine which calibration to load.")
)
externalWcsName = Field(
externalSkyWcsName = ChoiceField(
dtype=str,
doc="Type of external wcs. Currently supported is jointcal.",
default="jointcal"
doc="Type of external SkyWcs if `doApplyExternalSkyWcs` is True.",
default="jointcal",
allowed={
"jointcal": "Use jointcal_wcs"
}
)
skipTEx = Field(
dtype=bool, default=False,
Expand All @@ -129,12 +140,12 @@ class MatchedVisitMetricsTask(CmdLineTask):
keys as the `wcs` dataset (those used by the `calexp` dataset plus
`tract`). When `config.doApplyExternalPhotoCalib` is `True`, the
photometric calibration (`photoCalib`) is taken from
`config.externalPhotoCalibName`. Otherwise, the photometric calibration
is retrieved from the `calexp`. When `config.doApplyExternalWcs` is
`True`, the astrometric calibration (`wcs`) is taken from
`config.externalWcsName`. Otherwise, the astrometric calbration is
unchanged from the positions loaded from the `src` dataset. In all cases
the `tract` must be present.
`config.externalPhotoCalibName` via the `name_photoCalib` dataset. Otherwise,
the photometric calibration is retrieved from the `calexp`. When
`config.doApplyExternalSkyWcs` is `True`, the astrometric calibration is taken
from `config.externalSkyWcsName` with the `name_wcs` dataset. Otherwise, the
astrometric calbration is unchanged from the positions loaded from the
`src` dataset. In all cases the `tract` must be present.
"""

_DefaultName = "matchedVisitMetrics"
Expand All @@ -161,8 +172,8 @@ def run(self, butler, filterName, output, dataIds):
outputPrefix=output_prefix,
doApplyExternalPhotoCalib=self.config.doApplyExternalPhotoCalib,
externalPhotoCalibName=self.config.externalPhotoCalibName,
doApplyExternalWcs=self.config.doApplyExternalWcs,
externalWcsName=self.config.externalWcsName,
doApplyExternalSkyWcs=self.config.doApplyExternalSkyWcs,
externalSkyWcsName=self.config.externalSkyWcsName,
skipTEx=self.config.skipTEx,
verbose=self.config.verbose,
metrics_package=self.config.metricsRepository,
Expand Down
199 changes: 132 additions & 67 deletions python/lsst/validate/drp/matchreduce.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
from lsst.afw.table import (SourceCatalog, SchemaMapper, Field,
MultiMatch, SimpleRecord, GroupView,
SOURCE_IO_NO_FOOTPRINTS)
import lsst.afw.table as afwTable
from lsst.afw.fits import FitsError
from lsst.verify import Blob, Datum

Expand All @@ -42,9 +43,8 @@

def build_matched_dataset(repo, dataIds, matchRadius=None, safeSnr=50.,
doApplyExternalPhotoCalib=False, externalPhotoCalibName=None,
doApplyExternalWcs=False, externalWcsName=None,
uberCalAstrometryName=None, skipTEx=False,
skipNonSrd=False):
doApplyExternalSkyWcs=False, externalSkyWcsName=None,
skipTEx=False, skipNonSrd=False):
"""Construct a container for matched star catalogs from multple visits, with filtering,
summary statistics, and modelling.
Expand All @@ -64,14 +64,14 @@ def build_matched_dataset(repo, dataIds, matchRadius=None, safeSnr=50.,
Minimum median SNR for a match to be considered "safe".
doApplyExternalPhotoCalib : bool, optional
Apply external photoCalib to calibrate fluxes.
Default is False.
externalPhotoCalibName : str, optional
Type of external `PhotoCalib` to apply. Currently supported are jointcal,
fgcm, and fgcm_tract. Must be set if doApplyExternalPhotoCalib is True.
Default is None.
doApplyExternalWcs : bool, optional
fgcm, and fgcm_tract. Must be set if "doApplyExternalPhotoCalib" is True.
doApplyExternalSkyWcs : bool, optional
Apply external wcs to calibrate positions.
Default is False.
externalSkyWcsName : str, optional:
Type of external `wcs` to apply. Currently supported is jointcal.
Must be set if "doApplyExternalSkyWcs" is True.
skipTEx : `bool`, optional
Skip TEx calculations (useful for older catalogs that don't have
PsfShape measurements).
Expand Down Expand Up @@ -118,11 +118,18 @@ def build_matched_dataset(repo, dataIds, matchRadius=None, safeSnr=50.,
catalog tables.
*Not serialized.*
Raises
------
RuntimeError:
Raised if "doApplyExternalPhotoCalib" is True and "externalPhotoCalibName"
is None, or if "doApplyExternalSkyWcs" is True and "externalSkyWcsName" is
None.
"""
if doApplyExternalPhotoCalib and externalPhotoCalibName is None:
raise RuntimeError("Must set externalPhotoCalibName if doApplyExternalPhotoCalib is True.")
if doApplyExternalWcs and externalWcsName is None:
raise RuntimeError("Must set externalWcsName if doApplyExternalWcs is True.")
if doApplyExternalSkyWcs and externalSkyWcsName is None:
raise RuntimeError("Must set externalSkyWcsName if doApplyExternalSkyWcs is True.")

blob = Blob('MatchedMultiVisitDataset')

Expand All @@ -136,21 +143,21 @@ def build_matched_dataset(repo, dataIds, matchRadius=None, safeSnr=50.,
# Record important configuration
blob['doApplyExternalPhotoCalib'] = Datum(quantity=doApplyExternalPhotoCalib,
description=('Whether external photometric '
'calibrations were used'))
'calibrations were used.'))
blob['externalPhotoCalibName'] = Datum(quantity=externalPhotoCalibName,
description='Name of external PhotoCalib used')
blob['doApplyExternalWcs'] = Datum(quantity=doApplyExternalWcs,
description='Whether external wcs calibrations were used')
blob['externalWcsName'] = Datum(quantity=externalWcsName,
description='Name of external wcs used')
description='Name of external PhotoCalib dataset used.')
blob['doApplyExternalSkyWcs'] = Datum(quantity=doApplyExternalSkyWcs,
description='Whether external wcs calibrations were used.')
blob['externalSkyWcsName'] = Datum(quantity=externalSkyWcsName,
description='Name of external wcs dataset used.')

# Match catalogs across visits
blob._catalog, blob._matchedCatalog = \
_loadAndMatchCatalogs(repo, dataIds, matchRadius,
doApplyExternalPhotoCalib=doApplyExternalPhotoCalib,
externalPhotoCalibName=externalPhotoCalibName,
doApplyExternalWcs=doApplyExternalWcs,
externalWcsName=externalWcsName,
doApplyExternalSkyWcs=doApplyExternalSkyWcs,
externalSkyWcsName=externalSkyWcsName,
skipTEx=skipTEx, skipNonSrd=skipNonSrd)

blob.magKey = blob._matchedCatalog.schema.find("base_PsfFlux_mag").key
Expand All @@ -162,7 +169,7 @@ def build_matched_dataset(repo, dataIds, matchRadius=None, safeSnr=50.,

def _loadAndMatchCatalogs(repo, dataIds, matchRadius,
doApplyExternalPhotoCalib=False, externalPhotoCalibName=None,
doApplyExternalWcs=False, externalWcsName=None,
doApplyExternalSkyWcs=False, externalSkyWcsName=None,
skipTEx=False, skipNonSrd=False):
"""Load data from specific visits and returned a calibrated catalog matched
with a reference.
Expand All @@ -179,17 +186,14 @@ def _loadAndMatchCatalogs(repo, dataIds, matchRadius,
Radius for matching. Default is 1 arcsecond.
doApplyExternalPhotoCalib : bool, optional
Apply external photoCalib to calibrate fluxes.
Default is False.
externalPhotoCalibName : str, optional
Type of external `PhotoCalib` to apply. Currently supported are jointcal,
fgcm, and fgcm_tract. Must be set if doApplyExternalPhotoCalib is True.
Default is None.
doApplyExternalWcs : bool, optional
doApplyExternalSkyWcs : bool, optional
Apply external wcs to calibrate positions.
Default is False.
externalWcsName : str, optional
externalSkyWcsName : str, optional
Type of external `wcs` to apply. Currently supported is jointcal.
Must be set if doApplyExternalWcs is True. Default is None.
Must be set if "doApplyExternalWcs" is True.
skipTEx : `bool`, optional
Skip TEx calculations (useful for older catalogs that don't have
PsfShape measurements).
Expand All @@ -202,12 +206,19 @@ def _loadAndMatchCatalogs(repo, dataIds, matchRadius,
A new calibrated SourceCatalog.
matches : `lsst.afw.table.GroupView`
A GroupView of the matched sources.
Raises
------
RuntimeError:
Raised if "doApplyExternalPhotoCalib" is True and "externalPhotoCalibName"
is None, or if "doApplyExternalSkyWcs" is True and "externalSkyWcsName" is
None.
"""

if doApplyExternalPhotoCalib and externalPhotoCalibName is None:
raise RuntimeError("Must set externalPhotoCalibName if doApplyExternalPhotoCalib is True.")
if doApplyExternalWcs and externalWcsName is None:
raise RuntimeError("Must set externalWcsName if doApplyExternalWcs is True.")
if doApplyExternalSkyWcs and externalSkyWcsName is None:
raise RuntimeError("Must set externalSkyWcsName if doApplyExternalSkyWcs is True.")

# Following
# https://github.com/lsst/afw/blob/tickets/DM-3896/examples/repeatability.ipynb
Expand Down Expand Up @@ -272,46 +283,18 @@ def _loadAndMatchCatalogs(repo, dataIds, matchRadius,

for vId in dataIds:

if doApplyExternalPhotoCalib:
try:
photoCalib = butler.get('%s_photoCalib' % (externalPhotoCalibName), vId)
except (FitsError, dafPersist.NoResults) as e:
print(e)
print("Could not open external photometric calibration for ", vId)
print("Skipping this dataId.")
continue
else:
try:
photoCalib = butler.get('calexp_photoCalib', vId)
except (FitsError, dafPersist.NoResults) as e:
print(e)
print("Could not open calibrated image file for ", vId)
print("Skipping this dataId.")
except TypeError as te:
# DECam images that haven't been properly reformatted
# can trigger a TypeError because of a residual FITS header
# LTV2 which is a float instead of the expected integer.
# This generates an error of the form:
#
# lsst::pex::exceptions::TypeError: 'LTV2 has mismatched type'
#
# See, e.g., DM-2957 for details.
print(te)
print("Calibration image header information malformed.")
print("Skipping this dataId.")
continue
photoCalib = _loadPhotoCalib(butler, vId,
doApplyExternalPhotoCalib, externalPhotoCalibName)
if photoCalib is None:
continue

if doApplyExternalWcs:
try:
wcs = butler.get("%s_wcs" % (externalWcsName), vId)
except (FitsError, dafPersist.NoResults) as e:
print(e)
print("Could not open external WCS for ", vId)
print("Skipping this dataId.")
if doApplyExternalSkyWcs:
wcs = _loadExternalSkyWcs(butler, vId, externalSkyWcsName)
if wcs is None:
continue

# We don't want to put this above the first "if doApplyExternalPhotoCalib
# block" because we need to use the first `butler.get` above to quickly
# We don't want to put this above the first _loadPhotoCalib call
# because we need to use the first `butler.get` in there to quickly
# catch dataIDs with no usable outputs.
try:
# HSC supports these flags, which dramatically improve I/O
Expand All @@ -329,9 +312,8 @@ def _loadAndMatchCatalogs(repo, dataIds, matchRadius,
tmpCat['base_PsfFlux_snr'][:] = tmpCat['base_PsfFlux_instFlux'] \
/ tmpCat['base_PsfFlux_instFluxErr']

if doApplyExternalWcs:
for record in tmpCat:
record.updateCoord(wcs)
if doApplyExternalSkyWcs:
afwTable.wcsUtils.updateSourceCoords(wcs, tmpCat)
photoCalib.instFluxToMagnitude(tmpCat, "base_PsfFlux", "base_PsfFlux")
if not skipNonSrd:
tmpCat['slot_ModelFlux_snr'][:] = (tmpCat['slot_ModelFlux_instFlux'] /
Expand Down Expand Up @@ -455,3 +437,86 @@ def fullFilter(cat):
# These attributes are not serialized
blob.goodMatches = goodMatches
blob.safeMatches = safeMatches


def _loadPhotoCalib(butler, dataId, doApplyExternalPhotoCalib, externalPhotoCalibName):
"""
Load a photoCalib object.
Parameters
----------
butler: `lsst.daf.persistence.Butler`
dataId: Butler dataId `dict`
doApplyExternalPhotoCalib: `bool`
Apply external photoCalib to calibrate fluxes.
externalPhotoCalibName: `str`
Type of external `PhotoCalib` to apply. Currently supported are jointcal,
fgcm, and fgcm_tract. Must be set if "doApplyExternalPhotoCalib" is True.
Returns
-------
photoCalib: `lsst.afw.image.PhotoCalib` or None
photoCalib to apply. None if a suitable one was not found.
"""

photoCalib = None

if doApplyExternalPhotoCalib:
try:
photoCalib = butler.get(f"{externalPhotoCalibName}_photoCalib", dataId)
except (FitsError, dafPersist.NoResults) as e:
print(e)
print("Could not open external photometric calibration for ", dataId)
print("Skipping this dataId.")
photoCalib = None
else:
try:
photoCalib = butler.get('calexp_photoCalib', dataId)
except (FitsError, dafPersist.NoResults) as e:
print(e)
print("Could not open calibrated image file for ", dataId)
print("Skipping this dataId.")
except TypeError as te:
# DECam images that haven't been properly reformatted
# can trigger a TypeError because of a residual FITS header
# LTV2 which is a float instead of the expected integer.
# This generates an error of the form:
#
# lsst::pex::exceptions::TypeError: 'LTV2 has mismatched type'
#
# See, e.g., DM-2957 for details.
print(te)
print("Calibration image header information malformed.")
print("Skipping this dataId.")
photoCalib = None

return photoCalib


def _loadExternalSkyWcs(butler, dataId, externalSkyWcsName):
"""
Load a SkyWcs object.
Parameters
----------
butler: `lsst.daf.persistence.Butler`
dataId: Butler dataId `dict`
externalSkyWcsName: `str`
Type of external `SkyWcs` to apply. Currently supported is jointcal.
Must be not None if "doApplyExternalSkyWcs" is True.
Returns
-------
SkyWcs: `lsst.afw.geom.SkyWcs` or None
SkyWcs to apply. None if a suitable one was not found.
"""

try:
wcs = butler.get(f"{externalSkyWcsName}_wcs", dataId)
except (FitsError, dafPersist.NoResults) as e:
print(e)
print("Could not open external WCS for ", dataId)
print("Skipping this dataId.")
wcs = None

return wcs

0 comments on commit cae20c0

Please sign in to comment.