Skip to content

Commit

Permalink
fixup! Add PipelineTask that produces updated visit summary datasets.
Browse files Browse the repository at this point in the history
  • Loading branch information
@TallJimbo
TallJimbo committed Jan 9, 2023
1 parent 830a147 commit cee0825
Show file tree
Hide file tree
Showing 2 changed files with 68 additions and 63 deletions.
8 changes: 3 additions & 5 deletions ....drp.tasks/tasks/lsst.drp.tasks.update_visit_summary.UpdateVisitSummaryTask.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,19 @@
UpdateVisitSummaryTask
######################

``UpdateVisitSummaryTask`` combines updated versions of the non-trivial metadata used to characterize and calibrate a single-epoch into a single per-visit exposure catalog (``finalVisitSummary``).
``UpdateVisitSummaryTask`` combines updated versions of the various `~lsst.afw.image.Exposure` component objects used to characterize and calibrate a single-epoch image into a single per-visit exposure catalog (``finalVisitSummary``).
It also recomputes summary statistics to reflect these updates.

.. _lsst.drp.tasks.update_visit_summary.UpdateVisitSummary-summary:

Processing summary
==================

``UpdateVisitSummaryTask`` reads in an initial summary dataset of essentially the same form (``visitSummary``), then replaces the object fields of each
record with objects loaded from other (often optional) input datasets.
When an object field is replaced, any related summary statistics in the catalogs regular columns are also recomputed.
``UpdateVisitSummaryTask`` reads in the initial summary dataset of essentially the same form (``visitSummary``), then replaces the object fields of each record with objects loaded from other (often optional) input datasets that contain newer (often final) versions of those objects.
When an object field is replaced, any related summary statistics in the catalog's non-object columns are also recomputed.

See connection and ``run`` argument documentation for details.


.. _lsst.drp.tasks.update_visit_summary.UpdateVisitSummaryTask-api:

Python API summary
Expand Down
123 changes: 65 additions & 58 deletions python/lsst/drp/tasks/update_visit_summary.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,15 +37,14 @@

import astropy.table
import lsst.pipe.base.connectionTypes as cT

from lsst.geom import Angle, SpherePoint, Box2I, degrees
from lsst.afw.geom import SkyWcs
from lsst.afw.image import ExposureSummaryStats
from lsst.afw.math import BackgroundList
from lsst.afw.table import ExposureCatalog, ExposureRecord, SchemaMapper
from lsst.daf.butler import Butler, DeferredDatasetHandle, DatasetRef
from lsst.daf.butler import Butler, DatasetRef, DeferredDatasetHandle
from lsst.daf.butler.formatters.parquet import pandas_to_astropy
from lsst.pex.config import ConfigurableField, ChoiceField
from lsst.geom import Angle, Box2I, SpherePoint, degrees
from lsst.pex.config import ChoiceField, ConfigurableField
from lsst.pipe.base import (
ButlerQuantumContext,
InputQuantizedConnection,
Expand Down Expand Up @@ -112,15 +111,15 @@ def best_for_detector(
----------
detector_id : `int`
Detector ID; used to find the right row in the catalog or catalogs.
center : `SpherePoint` or `None`
center : `lsst.geom.SpherePoint` or `None`
Center of the detector in sky coordinates. If not provided, one
will be computed via `compute_center_for_detector_record`.
bbox : `lsst.geom.Box2I`, optional
Bounding box for the detector in its own pixel coordinates.
Returns
-------
tract_id: `int`
tract_id : `int`
ID of the tract that supplied this record, or `-1` if ``record`` is
`None` or if the input was not per-tract.
record : `lsst.afw.table.ExposureRecord` or `None`
Expand All @@ -134,12 +133,15 @@ def best_for_detector(
class PerTractInput(PossiblyMultipleInput):
"""Wrapper class for input `~lsst.afw.table.ExposureCatalog` datasets
that are per-tract.
This selects the best tract via the minimum average distance (on the sky)
from the detector's corners to the tract center.
"""

catalogs_by_tract: list[tuple[TractInfo, ExposureCatalog]]
"""List of tuples of catalogs and the tracts they correspond to
(`list` [ `tuple` [ `lsst.skymap.TractInfo`,
`lsst.afw.table.ExposureCatalog` ] ] ).
(`list` [`tuple` [`lsst.skymap.TractInfo`,
`lsst.afw.table.ExposureCatalog`]]).
"""

@classmethod
Expand All @@ -157,7 +159,7 @@ def load(
Butler proxy used in `~lsst.pipe.base.PipelineTask.runQuantum`.
sky_map : `lsst.skymap.BaseSkyMap`
Definition of tracts and patches.
refs : `~collections.abc.Iterable` [ `lsst.daf.butler.DatasetRef` ]
refs : `~collections.abc.Iterable` [`lsst.daf.butler.DatasetRef`]
References to the catalog datasets to load.
Returns
Expand All @@ -184,7 +186,7 @@ def best_for_detector(
bbox: Box2I | None = None,
) -> tuple[int, ExposureRecord | None]:
# Docstring inherited.
best: tuple[int, ExposureRecord | None] = (-1, None)
best_result: tuple[int, ExposureRecord | None] = (-1, None)
best_distance: Angle = float("inf") * degrees
for tract_info, catalog in self.catalogs_by_tract:
record = catalog.find(detector_id)
Expand All @@ -200,9 +202,9 @@ def best_for_detector(
center_for_record = center
center_distance = tract_info.ctr_coord.separation(center_for_record)
if best_distance > center_distance:
best = (tract_info.tract_id, record)
best_result = (tract_info.tract_id, record)
best_distance = center_distance
return best
return best_result


@dataclasses.dataclass
Expand Down Expand Up @@ -233,24 +235,21 @@ class UpdateVisitSummaryConnections(
"photoCalibName": "fgcm",
},
):
sky_map = cT.Input(
doc="Description of tract/patch geometry.",
name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
dimensions=("skymap",),
storageClass="SkyMap",
)
input_summary_schema = cT.InitInput(
doc="Schema for input_summary_catalog.",
name="visitSummary_schema",
storageClass="ExposureCatalog",
)
sky_map = cT.Input(
doc="Description of tract/patch geometry",
name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
storageClass="SkyMap",
dimensions=("skymap",),
)
input_summary_catalog = cT.Input(
doc="Visit summary table to load and modify.",
name="visitSummary",
dimensions=(
"instrument",
"visit",
),
dimensions=("instrument", "visit"),
storageClass="ExposureCatalog",
)
input_exposures = cT.Input(
Expand All @@ -265,7 +264,7 @@ class UpdateVisitSummaryConnections(
deferLoad=True,
)
psf_overrides = cT.Input(
doc="Visit-level catalog of PSFs to use.",
doc="Visit-level catalog of updated PSFs to use.",
name="finalized_psf_ap_corr_catalog",
dimensions=("instrument", "visit"),
storageClass="ExposureCatalog",
Expand All @@ -277,33 +276,33 @@ class UpdateVisitSummaryConnections(
storageClass="DataFrame",
)
ap_corr_overrides = cT.Input(
doc="Visit-level catalog of aperture correction maps to use.",
doc="Visit-level catalog of updated aperture correction maps to use.",
name="finalized_psf_ap_corr_catalog",
dimensions=("instrument", "visit"),
storageClass="ExposureCatalog",
)
photo_calib_overrides_tract = cT.Input(
doc="Visit-level catalog of photometric calibration objects to use.",
doc="Per-Tract visit-level catalog of updated photometric calibration objects to use.",
name="{photoCalibName}PhotoCalibCatalog",
dimensions=("instrument", "visit", "tract"),
storageClass="ExposureCatalog",
multiple=True,
)
photo_calib_overrides_global = cT.Input(
doc="Visit-level catalog of photometric calibration objects to use.",
doc="Global visit-level catalog of updated photometric calibration objects to use.",
name="{photoCalibName}PhotoCalibCatalog",
dimensions=("instrument", "visit"),
storageClass="ExposureCatalog",
)
wcs_overrides_tract = cT.Input(
doc="Per-tract visit-level catalog of astrometric calibration objects to use.",
doc="Per-tract visit-level catalog of updated astrometric calibration objects to use.",
name="{skyWcsName}SkyWcsCatalog",
dimensions=("instrument", "visit", "tract"),
storageClass="ExposureCatalog",
multiple=True,
)
wcs_overrides_global = cT.Input(
doc="Global visit-level catalog of astrometric calibration objects to use.",
doc="Global visit-level catalog of updated astrometric calibration objects to use.",
name="{skyWcsName}SkyWcsCatalog",
dimensions=("instrument", "visit"),
storageClass="ExposureCatalog",
Expand All @@ -325,7 +324,7 @@ class UpdateVisitSummaryConnections(
deferLoad=True,
)
output_summary_schema = cT.InitOutput(
doc="Schema of the output visit summary catalog",
doc="Schema of the output visit summary catalog.",
name="finalVisitSummary_schema",
storageClass="ExposureCatalog",
)
Expand Down Expand Up @@ -415,9 +414,9 @@ class UpdateVisitSummaryConfig(
},
# If needed, we could add options here to propagate the WCS from
# the input exposures and/or transfer WCS-based summary statistics
# them as well. Right now there's no use case for that, since the
# input visit is always produced after the last time we write a new
# Exposure.
# from them as well. Right now there's no use case for that, since
# the input visit summary is always produced after the last time we
# write a new Exposure.
},
default="input_summary",
optional=False,
Expand Down Expand Up @@ -446,14 +445,14 @@ class UpdateVisitSummaryConfig(
# If needed, we could add options here to propagate the PhotoCalib
# from the input exposures and/or transfer photometric calibration
# summary statistics them as well. Right now there's no use case
# for that, since the input visit is always produced after the last
# time we write a new Exposure.
# for that, since the input visit summary is always produced after
# the last time we write a new Exposure.
},
default="input_summary",
optional=False,
)
background_provider = ChoiceField(
doc="Which connection(s) and behavior to use when applying background overides.",
doc="Which connection(s) and behavior to use when applying background overrides.",
dtype=str,
allowed={
"input_summary": (
Expand Down Expand Up @@ -604,10 +603,11 @@ def run(
Parameters
----------
input_summary_catalog : `lsst.afw.table.ExposureCatalog`
Input catalog. A new row will be created for each output row in
each catalog, and any override parameter that is `None` will leave
the corresponding columns unchanged from those in this catalog.
input_exposures : `collections.abc.Mapping` [ `int`,
Input catalog. Each row in this catalog will be used to produce
a row in the output catalog. Any override parameter that is `None`
will leave the corresponding values unchanged from those in this
input catalog.
input_exposures : `collections.abc.Mapping` [`int`,
`lsst.daf.butler.DeferredDatasetHandle`]
Deferred-load objects that fetch `lsst.afw.image.Exposure`
instances. Only the image, mask, and variance are used; all other
Expand All @@ -631,7 +631,7 @@ def run(
wcs_overrides : `PossiblyMultipleInput`, optional
Catalog wrappers with attached `lsst.afw.geom.SkyWcs` objects
that supersede the input catalog's astrometric calibrations.
background_originals : `collections.abc.Mapping` [ `int`,
background_originals : `collections.abc.Mapping` [`int`,
`lsst.daf.butler.DeferredDatasetHandle`], optional
Deferred-load objects that fetch `lsst.afw.math.BackgroundList`
instances. These should correspond to the background already
Expand All @@ -640,13 +640,18 @@ def run(
``input_exposures`` has not been subtracted. If provided, all keys
in ``background_overrides`` must also be present in
``background_originals``.
background_overrides : `collections.abc.Mapping` [ `int`,
background_overrides : `collections.abc.Mapping` [`int`,
`lsst.daf.butler.DeferredDatasetHandle`], optional
Deferred-load objects that fetch `lsst.afw.math.BackgroundList`
instances. These should correspond to the background that should
now be subtracted from``input_exposures`` to yield the final
background-subtracted image.
Returns
-------
output_summary_catalog : `lsst.afw.table.ExposureCatalog`
Output visit summary catalog.
Notes
-----
If any override parameter is provided but does not have a value for a
Expand All @@ -659,24 +664,26 @@ def run(
output_summary_catalog = ExposureCatalog(self.schema)
output_summary_catalog.setMetadata(input_summary_catalog.getMetadata())
for input_record in input_summary_catalog:
detector = input_record.getId()
detector_id = input_record.getId()
output_record = output_summary_catalog.addNew()

# Make a new ExposureSummaryStats from the input record.
summary_stats = ExposureSummaryStats.from_record(input_record)

# Also copy the input record values to output record; this copies
# many of the same things (which will be overridden later by
# summary_stats.update_record), but also copies fields that aren't
# part of summary_stats, including the actual components like
# Psf, Wcs, etc.
# many of the same values just copied into `summary_stats` (which
# will be overridden later by summary_stats.update_record), but it
# also copies fields that aren't part of summary_stats, including
# the actual components like Psf, Wcs, etc.
output_record.assign(input_record, self.schema_mapper)

exposure = input_exposures[detector].get()
exposure = input_exposures[detector_id].get()
bbox = exposure.getBBox()

if wcs_overrides:
wcs_tract, wcs_record = wcs_overrides.best_for_detector(detector, bbox=bbox)
wcs_tract, wcs_record = wcs_overrides.best_for_detector(
detector_id, bbox=bbox
)
if wcs_record is not None:
wcs = wcs_record.getWcs()
else:
Expand All @@ -690,15 +697,13 @@ def run(
else:
wcs = input_record.getWcs()

center = compute_center_for_detector_record(output_record, bbox, wcs)

if psf_overrides:
if (psf_record := psf_overrides.find(detector)) is not None:
if (psf_record := psf_overrides.find(detector_id)) is not None:
psf = psf_record.getPsf()
else:
psf = None
output_record.setPsf(psf)
sources = psf_star_catalog[psf_star_catalog["detector"] == detector]
sources = psf_star_catalog[psf_star_catalog["detector"] == detector_id]
self.compute_summary_stats.update_psf_stats(
summary_stats,
psf,
Expand All @@ -709,16 +714,18 @@ def run(
)

if ap_corr_overrides:
if (ap_corr_record := ap_corr_overrides.find(detector)) is not None:
if (ap_corr_record := ap_corr_overrides.find(detector_id)) is not None:
ap_corr = ap_corr_record.getApCorrMap()
else:
ap_corr = None
output_record.setApCorrMap(ap_corr)

if photo_calib_overrides:
photo_calib_tract, photo_calib_record = photo_calib_overrides.best_for_detector(
detector, center=center
)
center = compute_center_for_detector_record(output_record, bbox, wcs)
(
photo_calib_tract,
photo_calib_record,
) = photo_calib_overrides.best_for_detector(detector_id, center=center)
if photo_calib_record is not None:
photo_calib = photo_calib_record.getPhotoCalib()
else:
Expand All @@ -731,10 +738,10 @@ def run(
)

if background_overrides is not None:
if (handle := background_overrides.get(detector)) is not None:
if (handle := background_overrides.get(detector_id)) is not None:
new_bkg = handle.get()
if background_originals is not None:
orig_bkg = background_originals[detector].get()
orig_bkg = background_originals[detector_id].get()
else:
orig_bkg = BackgroundList()

Expand Down

0 comments on commit cee0825

Please sign in to comment.