DM-34355: Improve analysis_drp scatterPlot unit test

python/lsst/analysis/drp/plotUtils.py

@@ -1,10 +1,35 @@
+# This file is part of analysis_drp.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
 import numpy as np
 import matplotlib.pyplot as plt
 import scipy.odr as scipyODR
+import matplotlib
 from matplotlib import colors
+from typing import List, Tuple
 
 from lsst.geom import Box2D, SpherePoint, degrees
 
+null_formatter = matplotlib.ticker.NullFormatter()
+
 
 def parsePlotInfo(dataId, runName, tableName, bands, plotName, SN):
     """Parse plot info from the dataId
@@ -128,6 +153,86 @@ def generateSummaryStatsVisit(cat, colName, visitSummaryTable, plotInfo):
     return visitInfoDict
 
 
+# Inspired by matplotlib.testing.remove_ticks_and_titles
+def get_and_remove_axis_text(ax) -> Tuple[List[str], List[np.ndarray]]:
+    """Remove text from an Axis and its children and return with line points.
+
+    Parameters
+    ----------
+    ax : `plt.Axis`
+        A matplotlib figure axis.
+
+    Returns
+    -------
+    texts : `List[str]`
+        A list of all text strings (title and axis/legend/tick labels).
+    line_xys : `List[numpy.ndarray]`
+        A list of all line ``_xy`` attributes (arrays of shape ``(N, 2)``).
+    """
+    line_xys = [line._xy for line in ax.lines]
+    texts = [text.get_text() for text in (ax.title, ax.xaxis.label, ax.yaxis.label)]
+    ax.set_title("")
+    ax.set_xlabel("")
+    ax.set_ylabel("")
+
+    try:
+        texts_legend = ax.get_legend().texts
+        texts.extend(text.get_text() for text in texts_legend)
+        for text in texts_legend:
+            text.set_alpha(0)
+    except AttributeError:
+        pass
+
+    for idx in range(len(ax.texts)):
+        texts.append(ax.texts[idx].get_text())
+        ax.texts[idx].set_text('')
+
+    ax.xaxis.set_major_formatter(null_formatter)
+    ax.xaxis.set_minor_formatter(null_formatter)
+    ax.yaxis.set_major_formatter(null_formatter)
+    ax.yaxis.set_minor_formatter(null_formatter)
+    try:
+        ax.zaxis.set_major_formatter(null_formatter)
+        ax.zaxis.set_minor_formatter(null_formatter)
+    except AttributeError:
+        pass
+    for child in ax.child_axes:
+        texts_child, lines_child = get_and_remove_axis_text(child)
+        texts.extend(texts_child)
+
+    return texts, line_xys
+
+
+def get_and_remove_figure_text(figure: plt.Figure):
+    """Remove text from a Figure and its Axes and return with line points.
+
+    Parameters
+    ----------
+    figure : `matplotlib.pyplot.Figure`
+        A matplotlib figure.
+
+    Returns
+    -------
+    texts : `List[str]`
+        A list of all text strings (title and axis/legend/tick labels).
+    line_xys : `List[numpy.ndarray]`, (N, 2)
+        A list of all line ``_xy`` attributes (arrays of shape ``(N, 2)``).
+    """
+    texts = [str(figure._suptitle)]
+    lines = []
+    figure.suptitle("")
+
+    texts.extend(text.get_text() for text in figure.texts)
+    figure.texts = []
+
+    for ax in figure.get_axes():
+        texts_ax, lines_ax = get_and_remove_axis_text(ax)
+        texts.extend(texts_ax)
+        lines.extend(lines_ax)
+
+    return texts, lines
+
+
 def addPlotInfo(fig, plotInfo):
     """Add useful information to the plot
 

python/lsst/analysis/drp/scatterPlot.py

@@ -1,7 +1,27 @@
+# This file is part of analysis_drp.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
 import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
-from scipy.stats import median_absolute_deviation as sigmaMad
 from matplotlib import gridspec
 from matplotlib.patches import Rectangle
 from matplotlib.path import Path
@@ -16,6 +36,10 @@
 
 from . import dataSelectors as dataSelectors
 from .plotUtils import generateSummaryStats, parsePlotInfo, addPlotInfo, mkColormap
+from .statistics import sigmaMad
+
+cmapPatch = plt.cm.coolwarm.copy()
+cmapPatch.set_bad(color="none")
 
 
 class ScatterPlotWithTwoHistsTaskConnections(pipeBase.PipelineTaskConnections,
@@ -480,8 +504,7 @@ def scatterPlotWithTwoHists(self, catPlot, plotInfo, sumStats, yLims=False, xLim
                 # Check which points are outside 3 sigma MAD of the median
                 # and plot these as points.
                 inside = threeSigMadPath.contains_points(np.array([xs, ys]).T)
-                points, = ax.plot(xs[~inside], ys[~inside], ".", ms=3, alpha=0.3, mfc=color, mec=color,
-                                  zorder=-1)
+                ax.plot(xs[~inside], ys[~inside], ".", ms=3, alpha=0.3, mfc=color, mec=color, zorder=-1)
 
                 # Add some stats text
                 xPos = 0.65 - 0.4*j
@@ -505,7 +528,7 @@ def scatterPlotWithTwoHists(self, catPlot, plotInfo, sumStats, yLims=False, xLim
                     histIm = ax.hexbin(xs[inside], ys[inside], gridsize=75, cmap=cmap, mincnt=1, zorder=-2)
 
             else:
-                points, = ax.plot(xs, ys, ".", ms=5, alpha=0.3, mfc=color, mec=color, zorder=-1)
+                ax.plot(xs, ys, ".", ms=5, alpha=0.3, mfc=color, mec=color, zorder=-1)
                 meds = np.array([np.nanmedian(ys)]*len(xs))
                 medLine, = ax.plot(xs, meds, color, label=f"Median: {np.nanmedian(ys):0.2f}", lw=0.8)
                 linesForLegend.append(medLine)
@@ -626,11 +649,9 @@ def scatterPlotWithTwoHists(self, catPlot, plotInfo, sumStats, yLims=False, xLim
             if dataId != "tract":
                 axCorner.annotate(dataId, (cenX, cenY), color="k", fontsize=4, ha="center", va="center")
 
-        cmapUse = plt.cm.coolwarm
         # Set the bad color to transparent and make a masked array
-        cmapUse.set_bad(color="none")
         colors = np.ma.array(colors, mask=np.isnan(colors))
-        collection = PatchCollection(patches, cmap=cmapUse)
+        collection = PatchCollection(patches, cmap=cmapPatch)
         collection.set_array(colors)
         axCorner.add_collection(collection)
 

python/lsst/analysis/drp/statistics.py

@@ -0,0 +1,25 @@
+# This file is part of {{ cookiecutter.package_name }}.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+from functools import partial
+from scipy.stats import median_abs_deviation
+
+sigmaMad = partial(median_abs_deviation, scale='normal')

tests/data/test_scatterPlot_lines/line_0.txt

@@ -0,0 +1,27 @@
+2.318975599480923222e+01 5.460330231375820631e+00
+2.341670284728129303e+01 -1.387906737519983835e+01
+2.364364969975335384e+01 -3.559599041721028811e+01
+2.387059655222541465e+01 3.352275069857313383e+01
+2.409754340469747547e+01 -2.998955226815880337e+01
+2.432449025716953628e+01 -2.120631841139974938e+01
+2.455143710964159709e+01 8.645521466728922633e+00
+2.477838396211365790e+01 -2.447865334854526509e+00
+2.500533081458571871e+01 2.420490043991918583e+00
+2.523227766705777952e+01 -1.717050372736395047e+01
+2.545922451952984034e+01 -1.676355177448130007e+01
+2.568617137200190115e+01 -2.236160825699329280e+01
+2.591311822447396196e+01 -3.535395379951111750e+01
+2.614006507694602277e+01 -3.324687996517283750e+01
+2.636701192941808358e+01 -4.003508059864202551e+01
+2.659395878189014439e+01 -5.358893537075815061e+01
+2.682090563436220521e+01 -4.358072788341793569e+01
+2.704785248683426602e+01 -5.329311143376358473e+01
+2.727479933930632683e+01 -8.125794733412661230e+01
+2.750174619177838764e+01 -9.843317313849553329e+01
+2.772869304425044845e+01 -9.682089219439760086e+01
+2.795563989672250926e+01 -1.580139125992552351e+02
+2.818258674919457007e+01 -2.146585123893451907e+02
+2.840953360166663089e+01 -3.417397926455567472e+02
+2.863648045413869170e+01 -5.247018383987622201e+02
+2.886342730661075251e+01 -7.227667402882893839e+02
+2.909037415908281332e+01 -9.040077383843936332e+02

tests/data/test_scatterPlot_lines/line_1.txt

@@ -0,0 +1,27 @@
+2.318975599480923222e+01 7.549180933239404112e+01
+2.341670284728129303e+01 2.887667121000537662e+01
+2.364364969975335384e+01 1.670131167918545145e+02
+2.387059655222541465e+01 1.501283119406452613e+02
+2.409754340469747547e+01 1.223544913098249083e+02
+2.432449025716953628e+01 9.944447959974674234e+01
+2.455143710964159709e+01 2.243313716695904247e+02
+2.477838396211365790e+01 2.099477366211035303e+02
+2.500533081458571871e+01 1.913411169507329816e+02
+2.523227766705777952e+01 2.640926969978064562e+02
+2.545922451952984034e+01 2.948970923138157332e+02
+2.568617137200190115e+01 3.012447806082679449e+02
+2.591311822447396196e+01 3.534677892948337785e+02
+2.614006507694602277e+01 2.798760089760722849e+02
+2.636701192941808358e+01 4.280251165808102769e+02
+2.659395878189014439e+01 3.894124767193444541e+02
+2.682090563436220521e+01 5.029210479666078299e+02
+2.704785248683426602e+01 5.375296776580513551e+02
+2.727479933930632683e+01 5.545543615052843052e+02
+2.750174619177838764e+01 6.619809855057369532e+02
+2.772869304425044845e+01 6.958744986318314432e+02
+2.795563989672250926e+01 7.423286673332409009e+02
+2.818258674919457007e+01 6.239492787565567369e+02
+2.840953360166663089e+01 4.268513856887565794e+02
+2.863648045413869170e+01 1.713450320404976992e+02
+2.886342730661075251e+01 -8.888761165192920544e+01
+2.909037415908281332e+01 -3.780888864567848486e+02

tests/data/test_scatterPlot_lines/line_10.txt

@@ -0,0 +1,32 @@
+2.181354332976140498e+01 -1.593782238423102626e+00
+2.206122830698027570e+01 -2.895857913923549631e+00
+2.230891328419914643e+01 -2.695662538025445087e+00
+2.255659826141801716e+01 -3.332363454887854459e+00
+2.280428323863688789e+01 3.895882288858842912e+00
+2.305196821585575861e+01 3.211235763906472584e+00
+2.329965319307462934e+01 -1.603802733036729933e+00
+2.354733817029350007e+01 -8.261044270835071757e+00
+2.379502314751237080e+01 4.396491053729079113e+00
+2.404270812473124153e+01 -7.907887167263538686e+00
+2.429039310195011225e+01 5.010444330146412995e+00
+2.453807807916898298e+01 -4.741577536590213526e+00
+2.478576305638785371e+01 -6.445022981154124864e-01
+2.503344803360672444e+01 -7.954336801354600084e-01
+2.528113301082559516e+01 -6.871405219587956026e+00
+2.552881798804446589e+01 3.304543067645937526e-01
+2.577650296526333662e+01 -9.633966137954530495e+00
+2.602418794248220735e+01 -2.245915058141534359e+00
+2.627187291970107808e+01 -1.157692171518753810e+01
+2.651955789691994880e+01 -1.161901220703498439e+01
+2.676724287413881953e+01 -1.887685684141260367e+01
+2.701492785135769026e+01 -1.189901469390264310e+01
+2.726261282857656099e+01 -1.562799163333039587e+01
+2.751029780579543171e+01 -4.757069259273905004e+01
+2.775798278301430244e+01 -2.647832351886947322e+01
+2.800566776023317317e+01 -5.369570151807323555e+01
+2.825335273745204390e+01 -4.701953251137069856e+01
+2.850103771467091462e+01 -6.773479841410079416e+01
+2.874872269188978535e+01 -1.019677329865693594e+02
+2.899640766910865608e+01 -2.005505075244045088e+02
+2.924409264632752681e+01 -3.340150967486544005e+02
+2.949177762354639754e+01 -5.241881949176878379e+02

tests/data/test_scatterPlot_lines/line_11.txt

@@ -0,0 +1,32 @@
+2.181354332976140498e+01 2.102712721845012922e+01
+2.206122830698027570e+01 4.589137934083974812e+01
+2.230891328419914643e+01 2.254982769074320004e+01
+2.255659826141801716e+01 4.610752348773205966e+01
+2.280428323863688789e+01 4.871538536328853297e+01
+2.305196821585575861e+01 4.681625721370182447e+01
+2.329965319307462934e+01 5.690419520232649830e+01
+2.354733817029350007e+01 6.093551086749592116e+01
+2.379502314751237080e+01 1.085854438069414556e+02
+2.404270812473124153e+01 9.124734788145582343e+01
+2.429039310195011225e+01 1.433970445313738082e+02
+2.453807807916898298e+01 1.388840463122639335e+02
+2.478576305638785371e+01 1.389283895976816723e+02
+2.503344803360672444e+01 1.960104714896250755e+02
+2.528113301082559516e+01 1.873499491585817225e+02
+2.552881798804446589e+01 1.880785759825917580e+02
+2.577650296526333662e+01 2.216166844061556276e+02
+2.602418794248220735e+01 2.071252519880671343e+02
+2.627187291970107808e+01 2.690978149680866522e+02
+2.651955789691994880e+01 2.667995652159485189e+02
+2.676724287413881953e+01 3.688420290117728655e+02
+2.701492785135769026e+01 4.273007879082677505e+02
+2.726261282857656099e+01 4.111478623808086468e+02
+2.751029780579543171e+01 4.594902048196925080e+02
+2.775798278301430244e+01 5.546419653557846914e+02
+2.800566776023317317e+01 5.837439483265497984e+02
+2.825335273745204390e+01 6.112975189065015229e+02
+2.850103771467091462e+01 6.838385450001728714e+02
+2.874872269188978535e+01 6.767115176992004990e+02
+2.899640766910865608e+01 5.017770627997672932e+02
+2.924409264632752681e+01 2.791531617571469042e+02
+2.949177762354639754e+01 -4.720166609330112806e+01

tests/data/test_scatterPlot_lines/line_12.txt

@@ -0,0 +1,32 @@
+2.949177762354639754e+01 -1.001174723742074548e+03
+2.924409264632752681e+01 -9.471833552544557051e+02
+2.899640766910865608e+01 -9.028780778485763676e+02
+2.874872269188978535e+01 -8.806469836723391609e+02
+2.850103771467091462e+01 -8.193081418283743460e+02
+2.825335273745204390e+01 -7.053365839292429200e+02
+2.800566776023317317e+01 -6.911353513626963831e+02
+2.775798278301430244e+01 -6.075986123935236947e+02
+2.751029780579543171e+01 -5.546315900051706649e+02
+2.726261282857656099e+01 -4.424038456474694385e+02
+2.701492785135769026e+01 -4.510988172960730367e+02
+2.676724287413881953e+01 -4.065957426945981297e+02
+2.651955789691994880e+01 -2.900375896300184877e+02
+2.627187291970107808e+01 -2.922516583984617569e+02
+2.602418794248220735e+01 -2.116170821043501746e+02
+2.577650296526333662e+01 -2.408846166820646886e+02
+2.552881798804446589e+01 -1.874176673690625421e+02
+2.528113301082559516e+01 -2.010927595977576345e+02
+2.503344803360672444e+01 -1.976013388498959955e+02
+2.478576305638785371e+01 -1.402173941939124973e+02
+2.453807807916898298e+01 -1.483672013854443321e+02
+2.429039310195011225e+01 -1.333761558710809823e+02
+2.404270812473124153e+01 -1.070631222159829008e+02
+2.379502314751237080e+01 -9.979246169948329737e+01
+2.354733817029350007e+01 -7.745759940916606467e+01
+2.329965319307462934e+01 -6.011180066839995817e+01
+2.305196821585575861e+01 -4.039378568588887930e+01
+2.280428323863688789e+01 -4.092362078557084715e+01
+2.255659826141801716e+01 -5.277225039750776858e+01
+2.230891328419914643e+01 -2.794115276679409021e+01
+2.206122830698027570e+01 -5.168309516868684739e+01
+2.181354332976140498e+01 -2.421469169529633447e+01