Add statistics.py

python/lsst/analysis/tools/__init__.py

@@ -37,4 +37,5 @@
 """
 
 from .interfaces import *
+from .statistics import *
 from .version import *  # Generated by sconsUtils

python/lsst/analysis/tools/actions/keyedData/stellarLocusFit.py

@@ -27,9 +27,9 @@
 
 import numpy as np
 from lsst.pex.config import DictField
-from scipy.stats import median_absolute_deviation as sigmaMad
 
 from ...interfaces import KeyedData, KeyedDataAction, KeyedDataSchema, Scalar, Vector
+from ...statistics import sigmaMad
 from ..plot.plotUtils import perpDistance, stellarLocusFit
 
 

python/lsst/analysis/tools/actions/keyedData/summaryStatistics.py

@@ -20,12 +20,10 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from __future__ import annotations
 
-from functools import partial
-
-import scipy.stats as sps
 from lsst.pex.config import Field
 
 from ...interfaces import KeyedData
+from ...statistics import sigmaMad
 from ..scalar import CountAction, MedianAction, SigmaMadAction
 from .keyedDataActions import KeyedScalars
 
@@ -34,8 +32,6 @@
     "SummaryStatisticAction",
 )
 
-sigmaMad = partial(sps.median_abs_deviation, scale="normal")  # type: ignore
-
 
 class SummaryStatisticAction(KeyedScalars):
     vectorKey = Field[str](doc="Column key to compute scalars")

python/lsst/analysis/tools/actions/plot/colorColorFitPlot.py

@@ -30,10 +30,10 @@
 from lsst.pex.config import Field, ListField
 from matplotlib.figure import Figure
 from matplotlib.patches import Rectangle
-from scipy.stats import median_absolute_deviation as sigmaMad
 from sklearn.neighbors import KernelDensity
 
 from ...interfaces import KeyedData, KeyedDataSchema, Scalar, Vector
+from ...statistics import sigmaMad
 from .plotUtils import addPlotInfo, mkColormap, perpDistance
 
 

python/lsst/analysis/tools/actions/plot/histPlot.py

@@ -31,9 +31,9 @@
 from matplotlib.figure import Figure
 from matplotlib.gridspec import GridSpec
 from matplotlib.patches import Rectangle
-from scipy.stats import median_absolute_deviation as sigmaMad
 
 from ...interfaces import KeyedData, KeyedDataSchema, PlotAction, Vector
+from ...statistics import sigmaMad
 from .plotUtils import addPlotInfo
 
 

python/lsst/analysis/tools/actions/plot/scatterplotWithTwoHists.py

@@ -18,6 +18,7 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
 from __future__ import annotations
 
 __all__ = ("ScatterPlotStatsAction", "ScatterPlotWithTwoHists")
@@ -39,7 +40,8 @@
 from mpl_toolkits.axes_grid1 import make_axes_locatable
 
 from ...interfaces import KeyedData, KeyedDataAction, KeyedDataSchema, PlotAction, Scalar, Vector
-from ..keyedData.summaryStatistics import SummaryStatisticAction, sigmaMad
+from ...statistics import nansigmaMad, sigmaMad
+from ..keyedData.summaryStatistics import SummaryStatisticAction
 from ..scalar import MedianAction
 from ..vector import MagColumnNanoJansky, SnSelector
 from .plotUtils import addPlotInfo, addSummaryPlot, generateSummaryStats, mkColormap
@@ -406,13 +408,13 @@ def _scatterPlot(
             # ensure the columns are actually array
             xs = np.array(xs)
             ys = np.array(ys)
-            sigMadYs = sigmaMad(ys, nan_policy="omit")
+            sigMadYs = nansigmaMad(ys)
             if len(xs) < 2:
                 (medLine,) = ax.plot(
                     xs, np.nanmedian(ys), color, label=f"Median: {np.nanmedian(ys):.2g}", lw=0.8
                 )
                 linesForLegend.append(medLine)
-                sigMads = np.array([sigmaMad(ys, nan_policy="omit")] * len(xs))
+                sigMads = np.array([nansigmaMad(ys)] * len(xs))
                 (sigMadLine,) = ax.plot(
                     xs,
                     np.nanmedian(ys) + 1.0 * sigMads,
@@ -596,7 +598,7 @@ def _scatterPlot(
                 meds = np.array([np.nanmedian(ys)] * len(xs))
                 (medLine,) = ax.plot(xs, meds, color, label=f"Median: {np.nanmedian(ys):0.3g}", lw=0.8)
                 linesForLegend.append(medLine)
-                sigMads = np.array([sigmaMad(ys, nan_policy="omit")] * len(xs))
+                sigMads = np.array([nansigmaMad(ys)] * len(xs))
                 (sigMadLine,) = ax.plot(
                     xs,
                     meds + 1.0 * sigMads,

python/lsst/analysis/tools/actions/plot/skyPlot.py

@@ -30,9 +30,9 @@
 from matplotlib.figure import Figure
 from matplotlib.patches import Rectangle
 from scipy.stats import binned_statistic_2d
-from scipy.stats import median_absolute_deviation as sigmaMad
 
 from ...interfaces import KeyedData, KeyedDataSchema, PlotAction, Scalar, Vector
+from ...statistics import nansigmaMad, sigmaMad
 from .plotUtils import addPlotInfo, extremaSort, mkColormap
 
 # from .plotUtils import generateSummaryStats, parsePlotInfo
@@ -129,7 +129,7 @@ def statsAndText(self, arr, mask=None):
         if mask is not None:
             arr = arr[mask]
         med = np.nanmedian(arr)
-        sigMad = sigmaMad(arr, nan_policy="omit")
+        sigMad = nansigmaMad(arr)
 
         statsText = (
             "Median: {:0.2f}\n".format(med)

python/lsst/analysis/tools/actions/scalar/scalarActions.py

@@ -4,10 +4,10 @@
 from typing import cast
 
 import numpy as np
-import scipy.stats as sps
 from lsst.pex.config import ChoiceField, Field
 
 from ...interfaces import KeyedData, KeyedDataSchema, Scalar, ScalarAction, Vector
+from ...statistics import nansigmaMad
 
 
 class MedianAction(ScalarAction):
@@ -54,10 +54,8 @@ def __call__(self, data: KeyedData, **kwargs) -> Scalar:
         return cast(
             Scalar,
             float(
-                sps.median_abs_deviation(
+                nansigmaMad(
                     data[self.vectorKey.format(**kwargs)][mask],  # type: ignore
-                    scale="normal",
-                    nan_policy="omit",
                 )
             ),
         )

python/lsst/analysis/tools/statistics.py

@@ -0,0 +1,32 @@
+# This file is part of analysis_tools.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from functools import partial
+
+import scipy.stats as sps
+
+__all__ = (
+    "nansigmaMad",
+    "sigmaMad",
+)
+
+nansigmaMad = partial(sps.median_abs_deviation, scale="normal", nan_policy="omit")
+sigmaMad = partial(sps.median_abs_deviation, scale="normal", nan_policy="propagate")