Merge branch 'main' into tickets/DM-39367

.gitignore

@@ -11,7 +11,7 @@ config.log
 
 # Built by sconsUtils
 version.py
-bin/
+/bin/
 
 # Pytest
 tests/.tests

bin.src/SConscript

@@ -0,0 +1,4 @@
+# -*- python -*-
+from lsst.sconsUtils import scripts
+
+scripts.BasicSConscript.shebang()

bin.src/verify_to_sasquatch.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+# This file is part of analysis_tools.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from lsst.analysis.tools.bin.verifyToSasquatch import main
+
+if __name__ == "__main__":
+    main()

doc/lsst.analysis.tools/action-types.rst

@@ -12,6 +12,7 @@ Scalar Actions
 
 .. automodapi:: lsst.analysis.tools.actions.scalar
     :no-inherited-members:
+    :no-main-docstr:
 
 Vector Actions
 ==============

doc/lsst.analysis.tools/getting-started.rst

@@ -15,7 +15,7 @@ and cloning analysis_tools. Then an overview of the package and follow that
 with walking through a series of examples of increasing complexity.
 
 Analysis tools is designed to work with any sort of keyed data but to make it 
-more intuitive initially we’ll talk about tables and column names.
+more intuitive initially we'll talk about tables and column names.
 
 Setting Up the Package and Getting Started With The Stack
 ---------------------------------------------------------
@@ -133,7 +133,7 @@ python/lsst/analysis/tools
 A Simple Plotting And Metric Example
 ====================================
 
-The first example we are going to look at is a very simple one and then we can build 
+We will start with a simple example and build
 up from there. We're going to start by adapting an existing plot and metric to our needs, we'll use a 
 sky plot to show the on sky distribution of the values of a column in the table.
 
@@ -146,7 +146,7 @@ then we’ll look more into how to define them. One of the great things about ac
 they allow us to only read in the columns we need from large tables.
 
 Each plot and/or metric is its own class, each one has a prep, process and produce section. 
-The prep section applies things like flag cuts and signal to noise cuts to the data. 
+The prep section manipulates input data, for example by performing flag cuts and signal to noise cuts.
 The process section builds the data required for the plot/metric, for example if the plot 
 is of a magnitude difference against a magnitude then the actions defined in the 
 process section will identify which flux column needs to be read in and turned into a magnitude. 

doc/lsst.analysis.tools/index.rst

@@ -48,6 +48,16 @@ The ``lsst.analysis.tools`` package is developed at `github.com/lsst/analysis_to
 
 Jira issues relating to this package can be found using the `analysis_tools <https://jira.lsstcorp.org/issues/?jql=project%20%3D%20DM%20AND%20component%20%3D%20analysis_tools>`_ component.
 
+.. _lsst.analysis_tools-scripts:
+
+Script reference
+================
+
+.. toctree::
+   :maxdepth: 1
+
+   scripts/verify_to_sasquatch.py
+
 .. _lsst.analysis_tools-pyapi:
 
 Python API Reference

doc/lsst.analysis.tools/scripts/verify_to_sasquatch.py.rst

@@ -0,0 +1,3 @@
+.. autoprogram:: lsst.analysis.tools.bin.verifyToSasquatch:makeParser()
+   :prog: verify_to_sasquatch.py
+   :groups:

pipelines/coaddDiffMatchedQualityExtended.yaml

@@ -41,7 +41,7 @@ tasks:
 
       atools.matchedRefPositionXDiffMetric: MatchedRefCoaddPositionMetric
       atools.matchedRefPositionXDiffMetric.variable: x
-      atools.matchedRefPositionXDiffMetric.applyContext: MatchedRefChiContext
+      atools.matchedRefPositionXDiffMetric.applyContext: MatchedRefDiffContext
 
       atools.matchedRefPositionXChiMetric: MatchedRefCoaddPositionMetric
       atools.matchedRefPositionXChiMetric.variable: x

pipelines/matchedVisitQualityCore.yaml

@@ -6,10 +6,10 @@ tasks:
     config:
       connections.outputName: matchedVisitCore
       atools.stellarPhotometricRepeatability: StellarPhotometricRepeatability
-      # atools.stellarPhotometricResiduals: StellarPhotometricResidualsFocalPlane
-      # atools.stellarAstrometricResidualsRA: StellarAstrometricResidualsRAFocalPlanePlot
-      # atools.stellarAstrometricResidualsDec: StellarAstrometricResidualsDecFocalPlanePlot
-      # atools.stellarAstrometricResidualStdDevRA: StellarAstrometricResidualStdDevRAFocalPlanePlot
-      # atools.stellarAstrometricResidualStdDevDec: StellarAstrometricResidualStdDevDecFocalPlanePlot
+      atools.stellarPhotometricResiduals: StellarPhotometricResidualsFocalPlane
+      atools.stellarAstrometricResidualsRA: StellarAstrometricResidualsRAFocalPlanePlot
+      atools.stellarAstrometricResidualsDec: StellarAstrometricResidualsDecFocalPlanePlot
+      atools.stellarAstrometricResidualStdDevRA: StellarAstrometricResidualStdDevRAFocalPlanePlot
+      atools.stellarAstrometricResidualStdDevDec: StellarAstrometricResidualStdDevDecFocalPlanePlot
       python: |
         from lsst.analysis.tools.atools import *

python/lsst/analysis/tools/actions/plot/__init__.py

@@ -1,6 +1,7 @@
 from .barPlots import *
 from .colorColorFitPlot import *
 from .diaSkyPlot import *
+from .focalPlanePlot import *
 from .histPlot import *
 from .multiVisitCoveragePlot import *
 from .rhoStatisticsPlot import *

python/lsst/analysis/tools/actions/plot/focalPlanePlot.py

@@ -150,12 +150,18 @@ def makePlot(
         fig : `matplotlib.figure.Figure`
             The resulting figure.
         """
-        fig = plt.figure(dpi=300)
-        ax = fig.add_subplot(111)
-
         if plotInfo is None:
             plotInfo = {}
 
+        if len(data["x"]) == 0:
+            noDataFig = Figure()
+            noDataFig.text(0.3, 0.5, "No data to plot after selectors applied")
+            noDataFig = addPlotInfo(noDataFig, plotInfo)
+            return noDataFig
+
+        fig = plt.figure(dpi=300)
+        ax = fig.add_subplot(111)
+
         detectorIds = np.unique(data["detector"])
         focalPlane_x = np.zeros(len(data["x"]))
         focalPlane_y = np.zeros(len(data["y"]))
@@ -170,8 +176,10 @@ def makePlot(
             focalPlane_x[detectorInd] = fp_x
             focalPlane_y[detectorInd] = fp_y
 
-        binsx = np.linspace(focalPlane_x.min(), focalPlane_x.max(), self.nBins)
-        binsy = np.linspace(focalPlane_y.min(), focalPlane_y.max(), self.nBins)
+        # Add an arbitrary small offset to bins to ensure that the minimum does
+        # not equal the maximum.
+        binsx = np.linspace(focalPlane_x.min() - 1e-5, focalPlane_x.max() + 1e-5, self.nBins)
+        binsy = np.linspace(focalPlane_y.min() - 1e-5, focalPlane_y.max() + 1e-5, self.nBins)
 
         statistic, x_edge, y_edge, binnumber = binned_statistic_2d(
             focalPlane_x, focalPlane_y, data["z"], statistic=self.statistic, bins=[binsx, binsy]

python/lsst/analysis/tools/actions/vector/calcBinnedStats.py

@@ -86,9 +86,15 @@ def name_sigmaMad(self):
         return f"{self.name_prefix}sigmaMad{self.name_suffix}"
 
     def __call__(self, data: KeyedData, **kwargs) -> KeyedData:
+        has_band = "band" in kwargs
+        kwargs_format = {}
+        if has_band:
+            kwargs_format["band"] = kwargs["band"]
+        prefix_band = f"{kwargs['band']}_" if has_band else ""
+
         results = {}
         mask = self.selector_range(data, **kwargs)
-        results[self.name_mask] = mask
+        results[self.name_mask.format(**kwargs_format)] = mask
         kwargs["mask"] = mask
 
         action = SummaryStatisticAction(vectorKey=self.key_vector)
@@ -97,14 +103,21 @@ def __call__(self, data: KeyedData, **kwargs) -> KeyedData:
         action.setDefaults()
 
         for name, value in action(data, **kwargs).items():
-            results[getattr(self, f"name_{name}")] = value
+            results[getattr(self, f"name_{name}").format(**kwargs_format)] = value
 
         values = cast(Vector, data[self.selector_range.key][mask])  # type: ignore
         valid = np.sum(np.isfinite(values)) > 0
-        results[self.name_select_maximum] = cast(Scalar, float(np.nanmax(values)) if valid else np.nan)
-        results[self.name_select_median] = cast(Scalar, float(np.nanmedian(values)) if valid else np.nan)
-        results[self.name_select_minimum] = cast(Scalar, float(np.nanmin(values)) if valid else np.nan)
-        results["range_maximum"] = self.selector_range.maximum
-        results["range_minimum"] = self.selector_range.minimum
+
+        results[self.name_select_maximum.format(**kwargs_format)] = cast(
+            Scalar, float(np.nanmax(values)) if valid else np.nan
+        )
+        results[self.name_select_median.format(**kwargs_format)] = cast(
+            Scalar, float(np.nanmedian(values)) if valid else np.nan
+        )
+        results[self.name_select_minimum.format(**kwargs_format)] = cast(
+            Scalar, float(np.nanmin(values)) if valid else np.nan
+        )
+        results[f"{prefix_band}range_maximum"] = self.selector_range.maximum
+        results[f"{prefix_band}range_minimum"] = self.selector_range.minimum
 
         return results

python/lsst/analysis/tools/atools/astrometricRepeatability.py

@@ -29,8 +29,7 @@
     "StellarAstrometricResidualsDecSkyPlot",
 )
 
-from ..actions.plot.focalPlanePlot import FocalPlanePlot
-from ..actions.plot.skyPlot import SkyPlot
+from ..actions.plot import FocalPlanePlot, SkyPlot
 from ..actions.vector import (
     BandSelector,
     ConvertFluxToMag,

python/lsst/analysis/tools/atools/diffMatched.py

@@ -33,7 +33,7 @@
 
 from ..actions.plot.scatterplotWithTwoHists import ScatterPlotStatsAction, ScatterPlotWithTwoHists
 from ..actions.vector.calcBinnedStats import CalcBinnedStatsAction
-from ..actions.vector.mathActions import ConstantValue, DivideVector, SubtractVector
+from ..actions.vector.mathActions import ConstantValue, DivideVector, MultiplyVector, SubtractVector
 from ..actions.vector.selectors import GalaxySelector, RangeSelector, StarSelector
 from ..actions.vector.vectorActions import ConvertFluxToMag, DownselectVector, LoadVector, VectorSelector
 from ..interfaces import AnalysisTool, KeyedData
@@ -103,7 +103,7 @@ def configureMetrics(
         self,
         unit: str | None = None,
         name_prefix: str | None = None,
-        name_suffix: str = "_mad_ref_mag{minimum}",
+        name_suffix: str = "_ref_mag{minimum}",
         unit_select: str = "mag",
     ):
         """Configure metric actions and return units.
@@ -151,6 +151,8 @@ def configureMetrics(
                 )
 
                 action.name_prefix = name_prefix.format(name_class=name_class)
+                if self.parameterizedBand:
+                    action.name_prefix = f"{{band}}_{action.name_prefix}"
                 action.name_suffix = name_suffix.format(minimum=minimum)
 
                 units.update(
@@ -178,7 +180,6 @@ def setDefaults(self):
                 )
 
     def __call__(self, data: KeyedData, **kwargs):
-        self._validate()
         return super().__call__(data=data, **kwargs)
 
 
@@ -249,6 +250,10 @@ def setDefaults(self):
 class MatchedRefCoaddDiffPositionTool(MatchedRefCoaddToolBase):
     """Base tool for diffs between reference and measured coadd astrometry."""
 
+    scale_factor = Field[float](
+        doc="The factor to multiply positions by (i.e. the pixel scale if coordinates have pixel units)",
+        default=200,
+    )
     variable = ChoiceField[str](
         doc="The astrometric variable to compute metrics for",
         allowed={
@@ -268,9 +273,12 @@ def _setPos(self):
 
     def matchedRefDiffContext(self):
         self._setPos()
-        self.process.buildActions.diff = SubtractVector(
-            actionA=self.process.buildActions.pos_meas,
-            actionB=self.process.buildActions.pos_ref,
+        self.process.buildActions.diff = MultiplyVector(
+            actionA=ConstantValue(value=self.scale_factor),
+            actionB=SubtractVector(
+                actionA=self.process.buildActions.pos_meas,
+                actionB=self.process.buildActions.pos_ref,
+            ),
         )
 
     def matchedRefChiContext(self):
@@ -302,14 +310,14 @@ class MatchedRefCoaddPositionMetric(MatchedRefCoaddDiffPositionTool, MatchedRefC
 
     def matchedRefDiffContext(self):
         super().matchedRefDiffContext()
-        self.unit = "pix"
+        self.unit = "mas"
         self.name_prefix = f"astrom_{self.variable}_{{name_class}}_diff_"
         self.produce.metric.units = self.configureMetrics()
 
     def matchedRefChiContext(self):
         super().matchedRefChiContext()
         self.unit = ""
-        self.name_prefix = f"astrom_{self.variable}_{{name_class}}_diff_"
+        self.name_prefix = f"astrom_{self.variable}_{{name_class}}_chi_"
         self.produce.metric.units = self.configureMetrics()
 
     def setDefaults(self):

python/lsst/analysis/tools/atools/photometricRepeatability.py

@@ -27,8 +27,7 @@
 
 from lsst.pex.config import Field
 
-from ..actions.plot.focalPlanePlot import FocalPlanePlot
-from ..actions.plot.histPlot import HistPanel, HistPlot, HistStatsPanel
+from ..actions.plot import FocalPlanePlot, HistPanel, HistPlot, HistStatsPanel
 from ..actions.scalar.scalarActions import CountAction, FracThreshold, MedianAction
 from ..actions.vector import (
     BandSelector,