DM-43404: Add diffim QA plots

pipelines/apDetectorVisitQualityCore.yaml

@@ -53,7 +53,7 @@ tasks:
       python: |
         from lsst.analysis.tools.atools import *
   diffimTaskCore:
-    class: lsst.analysis.tools.tasks.DiffimDetectorVisitAnalysisTask
+    class: lsst.analysis.tools.tasks.DiffimDetectorVisitMetricsAnalysisTask
     config:
       connections.outputName: diffimMetadata
 
@@ -75,11 +75,40 @@ tasks:
         templateCoveragePercent: percent
       python: |
         from lsst.analysis.tools.atools import DiffimMetadataMetricTool
+  diffimTaskPlots:
+    class: lsst.analysis.tools.tasks.DiffimDetectorVisitSpatiallySampledPlotsTask
+    config:
+      connections.outputName: diffimPlots
+      atools.diaFlagHistogram: DiffimSpatialMetricsHistPlot
+      atools.interpolatedDipoleDensity: DiffimSpatialMetricsInterpolatePlot
+      atools.interpolatedDipoleDensity.metricName: dipole_density
+      atools.interpolatedDipoleDensity.produce.plot.zAxisLabel: Number/degree^2
+      atools.interpolatedSourceDensity: DiffimSpatialMetricsInterpolatePlot
+      atools.interpolatedSourceDensity.metricName: source_density
+      atools.interpolatedSourceDensity.produce.plot.zAxisLabel: Number/degree^2
+      atools.interpolatedTemplateMedian: DiffimSpatialMetricsInterpolatePlot
+      atools.interpolatedTemplateMedian.metricName: template_value
+      atools.interpolatedTemplateMedian.produce.plot.zAxisLabel: nJy
+      atools.interpolatedScienceMedian: DiffimSpatialMetricsInterpolatePlot
+      atools.interpolatedScienceMedian.metricName: science_value
+      atools.interpolatedScienceMedian.produce.plot.zAxisLabel: nJy
+      atools.interpolatedDiffimMedian: DiffimSpatialMetricsInterpolatePlot
+      atools.interpolatedDiffimMedian.metricName: diffim_value
+      atools.interpolatedDiffimMedian.produce.plot.zAxisLabel: nJy
+      atools.interpolatedSciencePsfSize: DiffimSpatialMetricsInterpolatePlot
+      atools.interpolatedSciencePsfSize.metricName: science_psfSize
+      atools.interpolatedSciencePsfSize.produce.plot.zAxisLabel: pixels
+      atools.interpolatedTemplatePsfSize: DiffimSpatialMetricsInterpolatePlot
+      atools.interpolatedTemplatePsfSize.metricName: template_psfSize
+      atools.interpolatedTemplatePsfSize.produce.plot.zAxisLabel: pixels
+      python: |
+        from lsst.analysis.tools.atools import DiffimSpatialMetricsHistPlot, DiffimSpatialMetricsInterpolatePlot
 subsets:
   promptQaMetrics:
     subset:
       - analyzeAssocDiaSrcCore
       - analyzeTrailedDiaSrcCore
       - diffimTaskCore
+      - diffimTaskPlots
     description: >
       QA metrics to run in Prompt Processing.

python/lsst/analysis/tools/actions/plot/__init__.py

@@ -5,6 +5,7 @@
 from .focalPlanePlot import *
 from .gridPlot import *
 from .histPlot import *
+from .interpolateDetectorPlot import *
 from .matrixPlot import *
 from .multiVisitCoveragePlot import *
 from .propertyMapPlot import *

python/lsst/analysis/tools/actions/plot/interpolateDetectorPlot.py

@@ -0,0 +1,86 @@
+# This file is part of analysis_tools.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+__all__ = ("InterpolateDetectorMetricPlot",)
+
+import logging
+from typing import Mapping, Optional
+
+import matplotlib.pyplot as plt
+import numpy as np
+from lsst.pex.config import Field
+from matplotlib.figure import Figure
+from scipy.interpolate import CloughTocher2DInterpolator
+
+from ...interfaces import KeyedData, KeyedDataSchema, PlotAction, Vector
+from .plotUtils import addPlotInfo
+
+_LOG = logging.getLogger(__name__)
+
+
+class InterpolateDetectorMetricPlot(PlotAction):
+    """Interpolate metrics evaluated at locations across a detector."""
+
+    xAxisLabel = Field[str](doc="Label to use for the x axis.", default="x (pixel)", optional=True)
+    yAxisLabel = Field[str](doc="Label to use for the y axis.", default="y (pixel)", optional=True)
+    zAxisLabel = Field[str](doc="Label to use for the z axis.", optional=True)
+
+    xCoordSize = Field[int]("Dimensions for X direction field to interpolate", default=4000)
+    yCoordSize = Field[int]("Dimensions for Y direction field to interpolate", default=4072)
+    nGridPoints = Field[int]("N points in the grid for the field to interpolate", default=40)
+    gridMargin = Field[int]("Grid margins for the field to interpolate", default=20)
+
+    def getInputSchema(self) -> KeyedDataSchema:
+        base = []
+
+        base.append(("x", Vector))
+        base.append(("y", Vector))
+        base.append(("metricValues", Vector))
+
+        return base
+
+    def __call__(self, data: KeyedData, **kwargs) -> Mapping[str, Figure] | Figure:
+        return self.makePlot(data, **kwargs)
+
+    def makePlot(self, data: KeyedData, plotInfo: Optional[Mapping[str, str]] = None, **kwargs) -> Figure:
+
+        X = np.linspace(-self.gridMargin, self.xCoordSize + self.gridMargin, self.nGridPoints)
+        Y = np.linspace(-self.gridMargin, self.yCoordSize + self.gridMargin, self.nGridPoints)
+        meshgridX, meshgridY = np.meshgrid(X, Y)  # 2D grid for interpolation
+
+        interp = CloughTocher2DInterpolator(list(zip(data["x"], data["y"])), data["metricValues"])
+        Z = interp(meshgridX, meshgridY)
+
+        fig, ax = plt.subplots(1, 1, figsize=(8, 6))
+        pc = ax.pcolormesh(X, Y, Z, shading="auto")
+        ax.scatter(data["x"], data["y"], s=5, c="black")
+        cbar = fig.colorbar(pc)
+        cbar.set_label(self.zAxisLabel, rotation=270)
+        ax.set_xlabel(self.xAxisLabel)
+        ax.set_ylabel(self.yAxisLabel)
+        ax.set_aspect("equal", "box")
+
+        # add general plot info
+        if plotInfo is not None:
+            fig = addPlotInfo(fig, plotInfo)
+
+        return fig

python/lsst/analysis/tools/atools/__init__.py

@@ -9,6 +9,7 @@
 from .diaSolarSystemObjectMetrics import *
 from .diaSourceMetrics import *
 from .diaSourceTableTractMetrics import *
+from .diaSpatialMetricsPlots import *
 from .diffimMetadataMetrics import *
 from .diffMatched import *
 from .fluxMetrics import *

python/lsst/analysis/tools/atools/diaSpatialMetricsPlots.py

@@ -0,0 +1,95 @@
+# This file is part of analysis_tools.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+__all__ = ("DiffimSpatialMetricsHistPlot", "DiffimSpatialMetricsInterpolatePlot")
+
+from lsst.pex.config import Field
+
+from ..actions.plot.histPlot import HistPanel, HistPlot
+from ..actions.plot.interpolateDetectorPlot import InterpolateDetectorMetricPlot
+from ..actions.vector import LoadVector
+from ..interfaces import AnalysisTool
+
+
+class DiffimSpatialMetricsHistPlot(AnalysisTool):
+    """Create histograms of the fraction of pixels with certain mask planes
+    set.
+    """
+
+    parameterizedBand: bool = False
+
+    def setDefaults(self):
+        super().setDefaults()
+
+        self.process.buildActions.bad_mask_fraction = LoadVector(vectorKey="bad_mask_fraction")
+        self.process.buildActions.cr_mask_fraction = LoadVector(vectorKey="cr_mask_fraction")
+        self.process.buildActions.detected_mask_fraction = LoadVector(vectorKey="detected_mask_fraction")
+        self.process.buildActions.detected_negative_mask_fraction = LoadVector(
+            vectorKey="detected_negative_mask_fraction"
+        )
+        self.process.buildActions.intrp_mask_fraction = LoadVector(vectorKey="intrp_mask_fraction")
+        self.process.buildActions.no_data_mask_fraction = LoadVector(vectorKey="no_data_mask_fraction")
+        self.process.buildActions.sat_mask_fraction = LoadVector(vectorKey="sat_mask_fraction")
+        self.process.buildActions.sat_template_mask_fraction = LoadVector(
+            vectorKey="sat_template_mask_fraction"
+        )
+        self.process.buildActions.streak_mask_fraction = LoadVector(vectorKey="streak_mask_fraction")
+
+        self.produce.plot = HistPlot()
+
+        self.produce.plot.panels["panel_flags"] = HistPanel()
+        self.produce.plot.panels["panel_flags"].label = "Flagged pixel fraction"
+        self.produce.plot.panels["panel_flags"].bins = 20
+        self.produce.plot.panels["panel_flags"].rangeType = "fixed"
+        self.produce.plot.panels["panel_flags"].lowerRange = 0
+        self.produce.plot.panels["panel_flags"].upperRange = 1.0
+        self.produce.plot.panels["panel_flags"].hists = dict(
+            no_data_mask_fraction="No data",
+            sat_mask_fraction="Saturated",
+            bad_mask_fraction="Bad",
+            cr_mask_fraction="Cosmic ray",
+            detected_mask_fraction="Detected",
+            detected_negative_mask_fraction="Detected negative",
+            intrp_mask_fraction="Interpolated",
+            sat_template_mask_fraction="Saturated template",
+            streak_mask_fraction="Streak",
+        )
+
+
+class DiffimSpatialMetricsInterpolatePlot(AnalysisTool):
+    """Interpolate spatially-sampled metric values and create low-resolution
+    images of the result.
+    """
+
+    metricName = Field[str](doc="Metric name to interpolate", optional=False)
+    parameterizedBand: bool = False
+
+    def setDefaults(self):
+        super().setDefaults()
+
+        self.produce.plot = InterpolateDetectorMetricPlot()
+        self.process.buildActions.x = LoadVector()
+        self.process.buildActions.x.vectorKey = "x"
+        self.process.buildActions.y = LoadVector()
+        self.process.buildActions.y.vectorKey = "y"
+
+    def finalize(self):
+        self.process.buildActions.metricValues = LoadVector()
+        self.process.buildActions.metricValues.vectorKey = self.metricName

python/lsst/analysis/tools/tasks/__init__.py

@@ -8,7 +8,8 @@
 from .ccdVisitTableAnalysis import *
 from .diaObjectDetectorVisitAnalysis import *
 from .diaSourceTableTractAnalysis import *
-from .diffimTaskDetectorVisitAnalysis import *
+from .diffimTaskDetectorVisitMetricsAnalysis import *
+from .diffimTaskDetectorVisitSpatiallySampledAnalysis import *
 from .diffMatchedAnalysis import *
 from .gatherResourceUsage import *
 from .objectTableSurveyAnalysis import *

python/lsst/analysis/tools/tasks/diffimTaskDetectorVisitAnalysis.py → python/lsst/analysis/tools/tasks/diffimTaskDetectorVisitMetricsAnalysis.py

@@ -20,15 +20,15 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations
 
-__all__ = ("DiffimDetectorVisitAnalysisConfig", "DiffimDetectorVisitAnalysisTask")
+__all__ = ("DiffimDetectorVisitMetricsAnalysisConfig", "DiffimDetectorVisitMetricsAnalysisTask")
 
 import pandas as pd
 from lsst.pipe.base import NoWorkFound, connectionTypes
 
 from ..interfaces import AnalysisBaseConfig, AnalysisBaseConnections, AnalysisPipelineTask
 
 
-class DiffimDetectorVisitAnalysisConnections(
+class DiffimDetectorVisitMetricsAnalysisConnections(
     AnalysisBaseConnections,
     dimensions=("visit", "band", "detector"),
 ):
@@ -46,15 +46,15 @@ class DiffimDetectorVisitAnalysisConnections(
     )
 
 
-class DiffimDetectorVisitAnalysisConfig(
-    AnalysisBaseConfig, pipelineConnections=DiffimDetectorVisitAnalysisConnections
+class DiffimDetectorVisitMetricsAnalysisConfig(
+    AnalysisBaseConfig, pipelineConnections=DiffimDetectorVisitMetricsAnalysisConnections
 ):
     pass
 
 
-class DiffimDetectorVisitAnalysisTask(AnalysisPipelineTask):
-    ConfigClass = DiffimDetectorVisitAnalysisConfig
-    _DefaultName = "DiffimDetectorVisitAnalysis"
+class DiffimDetectorVisitMetricsAnalysisTask(AnalysisPipelineTask):
+    ConfigClass = DiffimDetectorVisitMetricsAnalysisConfig
+    _DefaultName = "DiffimDetectorVisitMetricsAnalysis"
 
     def runQuantum(self, butlerQC, inputRefs, outputRefs):
         inputs = butlerQC.get(inputRefs)

python/lsst/analysis/tools/tasks/diffimTaskDetectorVisitSpatiallySampledAnalysis.py

@@ -0,0 +1,67 @@
+# This file is part of analysis_tools.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+__all__ = ("DiffimDetectorVisitSpatiallySampledPlotsConfig", "DiffimDetectorVisitSpatiallySampledPlotsTask")
+
+from lsst.pipe.base import connectionTypes
+
+from ..interfaces import AnalysisBaseConfig, AnalysisBaseConnections, AnalysisPipelineTask
+
+
+class DiffimDetectorVisitSpatiallySampledPlotsConnections(
+    AnalysisBaseConnections,
+    dimensions=("visit", "band", "detector"),
+    defaultTemplates={"coaddName": "goodSeeing", "fakesType": ""},
+):
+    data = connectionTypes.Input(
+        doc="QA metrics evaluated in locations throughout the difference image.",
+        name="{fakesType}{coaddName}Diff_spatiallySampledMetrics",
+        storageClass="ArrowAstropy",
+        dimensions=("instrument", "visit", "detector"),
+        deferLoad=True,
+    )
+
+
+class DiffimDetectorVisitSpatiallySampledPlotsConfig(
+    AnalysisBaseConfig, pipelineConnections=DiffimDetectorVisitSpatiallySampledPlotsConnections
+):
+    pass
+
+
+class DiffimDetectorVisitSpatiallySampledPlotsTask(AnalysisPipelineTask):
+    ConfigClass = DiffimDetectorVisitSpatiallySampledPlotsConfig
+    _DefaultName = "DiffimDetectorVisitSpatiallySampledPlots"
+
+    def runQuantum(self, butlerQC, inputRefs, outputRefs):
+        # Docs inherited from base class.
+        inputs = butlerQC.get(inputRefs)
+        dataId = butlerQC.quantum.dataId
+        plotInfo = self.parsePlotInfo(inputs, dataId)
+        plotInfo["tableName"] += f", detector: {plotInfo['detector']}"
+        data = self.loadData(inputs["data"])
+
+        outputs = self.run(
+            data=data,
+            plotInfo=plotInfo,
+            bands=plotInfo["band"],
+        )
+        butlerQC.put(outputs, outputRefs)