Merge pull request #210 from lsst/tickets/DM-42888

DM-42888: Incorporate amp-to-amp correlation plots in analysis_tools for camera diagnostics

pipelines/amplifierQualityCore.yaml

@@ -0,0 +1,12 @@
+description: |
+  Matrix plots for amp-to-amp correlations on a detector.
+tasks:
+  plotAmpBiasCorrelation:
+    class: lsst.analysis.tools.tasks.ampToAmpCorrelationAnalysis.AmpToAmpCorrelationAnalysisTask
+    config:
+      connections.inputDataType: verifyBiasCorrelations
+      connections.outputName: ampBiasCorrelations
+      atools.imageCorrelationPlot: ImageCorrelationPlot
+      atools.overscanCorrelationPlot: OverscanCorrelationPlot
+      python: |
+        from lsst.analysis.tools.atools import *

python/lsst/analysis/tools/actions/plot/matrixPlot.py

@@ -80,7 +80,10 @@ class MatrixPlot(PlotAction):
     Note that when `component1Key` and `component2Key` are specified, the x and
     y tick values and labels will be dynamically configured, thereby
     eliminating the need for providing `x/yAxisTickValues` and
-    `x/yAxisTickLabels`.
+    `x/yAxisTickLabels`. When `componentGroup1Key` and `componentGroup2Key` are
+    specified, the x and y axis labels are dynamically updated to include the
+    group names, prefixed by `xAxisLabel` and `yAxisLabel` for a more
+    descriptive labeling.
     """
 
     inputDim = ChoiceField[int](
@@ -99,7 +102,8 @@ class MatrixPlot(PlotAction):
     )
 
     matrixOrigin = ChoiceField[str](
-        doc="Determines the starting corner ('upper', 'lower') for matrix plots.",
+        doc="Determines the starting corner ('upper', 'lower') for matrix plots. It only affects the visual "
+        "appearance of the plot.",
         default="upper",
         allowed={
             "upper": "The origin is at the upper left corner.",
@@ -109,15 +113,29 @@ class MatrixPlot(PlotAction):
     )
 
     component1Key = Field[str](
-        doc="The key to access a list of names for the first component set in a correlation analysis. This "
-        "will be used to determine x-axis tick values and labels.",
+        doc="The key to access a list of names for the first set of components in a correlation analysis. "
+        "This will be used to determine x-axis tick values and tick labels.",
         default=None,
         optional=True,
     )
 
     component2Key = Field[str](
-        doc="The key to access a list of names for the second component set in a correlation analysis. This "
-        "will be used to determine y-axis tick values and labels.",
+        doc="The key to access a list of names for the second set of components in a correlation analysis. "
+        "This will be used to determine y-axis tick values and tick labels.",
+    )
+
+    componentGroup1Key = Field[str](
+        doc="The key to access a list of group names for the first set of components in a correlation "
+        "analysis. This will be used to determine the x-axis label.",
+        default=None,
+        optional=True,
+    )
+
+    componentGroup2Key = Field[str](
+        doc="The key to access a list of group names for the second set of components in a correlation "
+        "analysis. This will be used to determine the y-axis label.",
+        default=None,
+        optional=True,
     )
 
     xAxisLabel = Field[str](
@@ -374,11 +392,23 @@ def makePlot(self, data: KeyedData, plotInfo: Mapping[str, str] | None = None, *
         if self.title:
             ax.set_title(self.title, fontsize=self.titleFontSize)
 
-        if self.xAxisLabel:
-            ax.set_xlabel(self.xAxisLabel, fontsize=self.axisLabelFontSize)
+        if self.componentGroup1Key is not None and self.componentGroup2Key is not None:
+            componentGroup1 = set(data[self.componentGroup1Key])
+            componentGroup2 = set(data[self.componentGroup2Key])
+            if len(componentGroup1) != 1 or len(componentGroup2) != 1:
+                raise ValueError(
+                    f"Each column specified by {self.componentGroup1Key} and {self.componentGroup2Key} must "
+                    "contain identical values within itself, but they do not."
+                )
+            else:
+                xAxisLabel = self.xAxisLabel + str(componentGroup1.pop())
+                yAxisLabel = self.yAxisLabel + str(componentGroup2.pop())
+        else:
+            xAxisLabel = self.xAxisLabel
+            yAxisLabel = self.yAxisLabel
 
-        if self.yAxisLabel:
-            ax.set_ylabel(self.yAxisLabel, fontsize=self.axisLabelFontSize)
+        ax.set_xlabel(xAxisLabel, fontsize=self.axisLabelFontSize)
+        ax.set_ylabel(yAxisLabel, fontsize=self.axisLabelFontSize)
 
         # Set the colorbar and draw the image.
         norm = ImageNormalize(vmin=vrange[0], vmax=vrange[1])
@@ -403,10 +433,10 @@ def makePlot(self, data: KeyedData, plotInfo: Mapping[str, str] | None = None, *
         shift = 0.5 if self.setPositionsAtPixelBoundaries else 0
 
         if self.component1Key is not None and self.component2Key is not None:
-            xAxisTickValues = np.arange(matrix.shape[0] + shift)
-            yAxisTickValues = np.arange(matrix.shape[1] + shift)
-            xAxisTickLabels = {key + shift: str(val) for key, val in zip(range(matrix.shape[0]), comp1[0, :])}
-            yAxisTickLabels = {key + shift: str(val) for key, val in zip(range(matrix.shape[1]), comp2[:, 0])}
+            xAxisTickValues = np.arange(matrix.shape[1] + shift)
+            yAxisTickValues = np.arange(matrix.shape[0] + shift)
+            xAxisTickLabels = {key + shift: str(val) for key, val in zip(range(matrix.shape[1]), comp1[0, :])}
+            yAxisTickLabels = {key + shift: str(val) for key, val in zip(range(matrix.shape[0]), comp2[:, 0])}
         else:
             xAxisTickValues = self.xAxisTickValues
             yAxisTickValues = self.yAxisTickValues
@@ -550,6 +580,7 @@ def makePlot(self, data: KeyedData, plotInfo: Mapping[str, str] | None = None, *
                         mpl_path_effects.Normal(),
                     ]
                 )
+
         # Add plot info if provided.
         if plotInfo is not None:
             fig = addPlotInfo(fig, plotInfo)

python/lsst/analysis/tools/atools/__init__.py

@@ -1,3 +1,4 @@
+from .amplifierCorrelation import *
 from .astrometricRepeatability import *
 from .calibration import *
 from .coveragePlots import *

python/lsst/analysis/tools/atools/amplifierCorrelation.py

@@ -0,0 +1,105 @@
+# This file is part of analysis_tools.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+__all__ = (
+    "ImageCorrelationPlot",
+    "OverscanCorrelationPlot",
+)
+
+from ..actions.plot.matrixPlot import MatrixPlot
+from ..actions.vector import LoadVector
+from ..interfaces import AnalysisTool
+
+
+class BaseCorrelationPlot(AnalysisTool):
+    """Base class for correlation plots of amplifier biases."""
+
+    # Do not iterate over multiple bands in a parameterized manner because this
+    # AnalysisTool does not support band as a name parameter.
+    parameterizedBand: bool = False
+
+    def setupPlot(self, matrixKey: str, title: str):
+        """
+        Set up a matrix plot action.
+
+        Parameters
+        ----------
+        matrixKey : `str`
+            Key for the matrix being plotted.
+        title : `str`
+            Title for the plot.
+
+        Returns
+        -------
+        action : `~lsst.analysis.tools.actions.plot.matrixPlot.MatrixPlot`
+            The resulting plot action.
+        """
+        action = MatrixPlot()
+        action.matrixKey = matrixKey
+        action.component1Key = "ampComp"
+        action.component2Key = "ampName"
+        action.componentGroup1Key = "detectorComp"
+        action.componentGroup2Key = "detector"
+        action.setPositionsAtPixelBoundaries = True
+        action.hideMinorTicks = ["x", "y"]
+        action.tickLabelsFontSize = 6
+        action.title = title
+        action.titleFontSize = 9
+        action.xAxisLabel = "Amplifiers in detector "
+        action.yAxisLabel = "Amplifiers in detector "
+        action.colorbarLabel = "Correlation value"
+        action.colorbarLabelFontSize = 9
+        action.colorbarTickLabelFontSize = 7.5
+        return action
+
+    def setDefaults(self):
+        super().setDefaults()
+
+        # Initialize plot with specific parameters.
+        action = self.setupPlot(matrixKey=self.matrixKey, title=self.plotTitle)
+
+        # Load the relevant columns.
+        for key in (
+            action.matrixKey,
+            action.component1Key,
+            action.component2Key,
+            action.componentGroup1Key,
+            action.componentGroup2Key,
+        ):
+            setattr(self.process.buildActions, key, LoadVector(vectorKey=key))
+
+        # Provide the plot action to the AnalysisTool.
+        self.produce.plot = action
+
+
+class ImageCorrelationPlot(BaseCorrelationPlot):
+    """Plot image correlation of amplifier biases."""
+
+    matrixKey = "imageCorr"
+    plotTitle = "Image correlations"
+
+
+class OverscanCorrelationPlot(BaseCorrelationPlot):
+    """Plot overscan correlation of amplifier biases."""
+
+    matrixKey = "overscanCorr"
+    plotTitle = "Overscan correlations"

python/lsst/analysis/tools/tasks/__init__.py

@@ -1,3 +1,4 @@
+from .ampToAmpCorrelationAnalysis import *
 from .assocDiaSrcDetectorVisitAnalysis import *
 from .associatedSourcesTractAnalysis import *
 from .astrometricCatalogMatch import *

python/lsst/analysis/tools/tasks/ampToAmpCorrelationAnalysis.py

@@ -0,0 +1,57 @@
+# This file is part of analysis_tools.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+__all__ = ("AmpToAmpCorrelationAnalysisConfig", "AmpToAmpCorrelationAnalysisTask")
+
+from lsst.pipe.base import connectionTypes as ct
+
+from ..interfaces import AnalysisBaseConfig, AnalysisBaseConnections, AnalysisPipelineTask
+
+
+class AmpToAmpCorrelationAnalysisConnections(
+    AnalysisBaseConnections,
+    dimensions=("instrument",),
+    defaultTemplates={
+        "inputDataType": "verifyBiasCorrelations",
+        "outputName": "biasCorrelations",
+    },
+):
+    data = ct.Input(
+        doc="Detector-based amplifier bias correlations.",
+        name="{inputDataType}",
+        storageClass="ArrowAstropy",
+        deferLoad=True,
+        dimensions=("instrument",),
+    )
+
+
+class AmpToAmpCorrelationAnalysisConfig(
+    AnalysisBaseConfig, pipelineConnections=AmpToAmpCorrelationAnalysisConnections
+):
+    pass
+
+
+class AmpToAmpCorrelationAnalysisTask(AnalysisPipelineTask):
+    """Task to analyze the correlation between amplifier biases."""
+
+    ConfigClass = AmpToAmpCorrelationAnalysisConfig
+    _DefaultName = "amplifierAnalysisTask"