Structural changes to layout and function

This is a catch-all commit for many structural changes to the
package. This covers unifying analysisPlots and analysisMetrics
as well as laying the ground work for dispatching Metrics to
a time series database. There are a few other quality of life
changes that were made during the refactoring process such as
the introduction of a finalize method.

mypy.ini

@@ -3,6 +3,7 @@ show_error_codes = True
 warn_unused_configs = True
 warn_redundant_casts = True
 plugins = pydantic.mypy
+exclude = python/lsst/analysis/tools/atools
 
 [mypy-matplotlib.*]
 ignore_missing_imports = True

pipelines/apCcdVisitQualityCore.yaml

@@ -4,12 +4,11 @@ tasks:
   analyzeDiaSourceTableCore:
     class: lsst.analysis.tools.tasks.DiaSourceTableCcdVisitAnalysisTask
     config:
-      metrics.numDiaSources: NumDiaSourcesMetric
-      metrics.numDiaSourcesAll: NumDiaSourcesAllMetric
-      metrics.numDipoles: NumDipolesMetric
-      metrics.numSsObjects: NumSsObjectsMetric
+      atools.numDiaSources: NumDiaSourcesMetric
+      atools.numDiaSourcesAll: NumDiaSourcesAllMetric
+      atools.numDipoles: NumDipolesMetric
+      atools.numSsObjects: NumSsObjectsMetric
       connections.outputName: diaSourceTableCore
-      plots.simpleDiaPlot: SimpleDiaPlot
+      atools.simpleDiaPlot: SimpleDiaPlot
       python: |
-        from lsst.analysis.tools.analysisMetrics import *
-        from lsst.analysis.tools.analysisPlots import *
+        from lsst.analysis.tools.atools import *

pipelines/coaddColumnValidate.yaml

@@ -5,12 +5,12 @@ tasks:
     class: lsst.analysis.tools.tasks.ObjectTableTractAnalysisTask
     config:
       connections.outputName: objectTableColumnValidate
-      metrics.validPsfFluxMetric: ValidFracColumnMetric
-      metrics.validPsfFluxMetric.vectorKey: 'psfFlux'
-      metrics.validPsfFluxMetric.applyContext: CoaddContext
-      metrics.validCmodelFluxMetric: ValidFracColumnMetric
-      metrics.validCmodelFluxMetric.vectorKey: 'cModelFlux'
-      metrics.validCmodelFluxMetric.applyContext: CoaddContext
+      atools.validPsfFluxMetric: ValidFracColumnMetric
+      atools.validPsfFluxMetric.vectorKey: 'psfFlux'
+      atools.validPsfFluxMetric.applyContext: CoaddContext
+      atools.validCmodelFluxMetric: ValidFracColumnMetric
+      atools.validCmodelFluxMetric.vectorKey: 'cModelFlux'
+      atools.validCmodelFluxMetric.applyContext: CoaddContext
       python: |
-        from lsst.analysis.tools.analysisMetrics import *
+        from lsst.analysis.tools.atools import *
         from lsst.analysis.tools.contexts import *

pipelines/coaddDiffMatchedQualityExtended.yaml

@@ -7,62 +7,62 @@ tasks:
       connections.outputName: diff_matched_truth_summary_objectTable_tract
 
       # plots
-      plots.matchedRefCModelMagDiffPlot: MatchedRefCoaddCModelFluxPlot
-      plots.matchedRefCModelMagDiffPlot.applyContext: MatchedRefDiffContext
+      atools.matchedRefCModelMagDiffPlot: MatchedRefCoaddCModelFluxPlot
+      atools.matchedRefCModelMagDiffPlot.applyContext: MatchedRefDiffContext
 
-      plots.matchedRefCModelFluxChiPlot: MatchedRefCoaddCModelFluxPlot
-      plots.matchedRefCModelFluxChiPlot.applyContext: MatchedRefChiContext
+      atools.matchedRefCModelFluxChiPlot: MatchedRefCoaddCModelFluxPlot
+      atools.matchedRefCModelFluxChiPlot.applyContext: MatchedRefChiContext
 
       # TODO: Can this be a one liner?
-      plots.matchedRefPositionXDiffPlot: MatchedRefCoaddPositionPlot
+      atools.matchedRefPositionXDiffPlot: MatchedRefCoaddPositionPlot
       # TODO: variable must be defined before applyContext; can this be enforced?
       # (the resulting error if not is not very informative)
-      plots.matchedRefPositionXDiffPlot.variable: x
-      plots.matchedRefPositionXDiffPlot.applyContext: MatchedRefDiffContext
+      atools.matchedRefPositionXDiffPlot.variable: x
+      atools.matchedRefPositionXDiffPlot.applyContext: MatchedRefDiffContext
 
-      plots.matchedRefPositionXChiPlot: MatchedRefCoaddPositionPlot
-      plots.matchedRefPositionXChiPlot.variable: x
-      plots.matchedRefPositionXChiPlot.applyContext: MatchedRefChiContext
+      atools.matchedRefPositionXChiPlot: MatchedRefCoaddPositionPlot
+      atools.matchedRefPositionXChiPlot.variable: x
+      atools.matchedRefPositionXChiPlot.applyContext: MatchedRefChiContext
 
-      plots.matchedRefPositionYDiffPlot: MatchedRefCoaddPositionPlot
-      plots.matchedRefPositionYDiffPlot.variable: y
-      plots.matchedRefPositionYDiffPlot.applyContext: MatchedRefDiffContext
+      atools.matchedRefPositionYDiffPlot: MatchedRefCoaddPositionPlot
+      atools.matchedRefPositionYDiffPlot.variable: y
+      atools.matchedRefPositionYDiffPlot.applyContext: MatchedRefDiffContext
 
-      plots.matchedRefPositionYChiPlot: MatchedRefCoaddPositionPlot
-      plots.matchedRefPositionYChiPlot.variable: y
-      plots.matchedRefPositionYChiPlot.applyContext: MatchedRefChiContext
+      atools.matchedRefPositionYChiPlot: MatchedRefCoaddPositionPlot
+      atools.matchedRefPositionYChiPlot.variable: y
+      atools.matchedRefPositionYChiPlot.applyContext: MatchedRefChiContext
 
       # metrics
-      metrics.matchedRefCModelMagDiffMetric: MatchedRefCoaddCModelFluxMetric
-      metrics.matchedRefCModelMagDiffMetric.applyContext: MatchedRefDiffContext
+      atools.matchedRefCModelMagDiffMetric: MatchedRefCoaddCModelFluxMetric
+      atools.matchedRefCModelMagDiffMetric.applyContext: MatchedRefDiffContext
 
-      metrics.matchedRefCModelFluxChiMetric: MatchedRefCoaddCModelFluxMetric
-      metrics.matchedRefCModelFluxChiMetric.applyContext: MatchedRefChiContext
+      atools.matchedRefCModelFluxChiMetric: MatchedRefCoaddCModelFluxMetric
+      atools.matchedRefCModelFluxChiMetric.applyContext: MatchedRefChiContext
 
-      metrics.matchedRefPositionXDiffMetric: MatchedRefCoaddPositionMetric
-      metrics.matchedRefPositionXDiffMetric.variable: x
-      metrics.matchedRefPositionXDiffMetric.applyContext: MatchedRefChiContext
+      atools.matchedRefPositionXDiffMetric: MatchedRefCoaddPositionMetric
+      atools.matchedRefPositionXDiffMetric.variable: x
+      atools.matchedRefPositionXDiffMetric.applyContext: MatchedRefChiContext
 
-      metrics.matchedRefPositionXChiMetric: MatchedRefCoaddPositionMetric
-      metrics.matchedRefPositionXChiMetric.variable: x
-      metrics.matchedRefPositionXChiMetric.applyContext: MatchedRefChiContext
+      atools.matchedRefPositionXChiMetric: MatchedRefCoaddPositionMetric
+      atools.matchedRefPositionXChiMetric.variable: x
+      atools.matchedRefPositionXChiMetric.applyContext: MatchedRefChiContext
 
-      metrics.matchedRefPositionYDiffMetric: MatchedRefCoaddPositionMetric
-      metrics.matchedRefPositionYDiffMetric.variable: y
-      metrics.matchedRefPositionYDiffMetric.applyContext: MatchedRefDiffContext
+      atools.matchedRefPositionYDiffMetric: MatchedRefCoaddPositionMetric
+      atools.matchedRefPositionYDiffMetric.variable: y
+      atools.matchedRefPositionYDiffMetric.applyContext: MatchedRefDiffContext
 
-      metrics.matchedRefPositionYChiMetric: MatchedRefCoaddPositionMetric
-      metrics.matchedRefPositionYChiMetric.variable: y
-      metrics.matchedRefPositionYChiMetric.applyContext: MatchedRefChiContext
+      atools.matchedRefPositionYChiMetric: MatchedRefCoaddPositionMetric
+      atools.matchedRefPositionYChiMetric.variable: y
+      atools.matchedRefPositionYChiMetric.applyContext: MatchedRefChiContext
 
 
       python: |
-        from lsst.analysis.tools.analysisMetrics.diffMatchedMetrics import (
+        from lsst.analysis.tools.atools.diffMatchedMetrics import (
           MatchedRefCoaddCModelFluxMetric,
           MatchedRefCoaddPositionMetric,
         )
 
-        from lsst.analysis.tools.analysisPlots.diffMatchedPlots import (
+        from lsst.analysis.tools.atools.diffMatchedPlots import (
           MatchedRefCoaddCModelFluxPlot,
           MatchedRefCoaddPositionPlot,
         )

pipelines/coaddQualityCore.yaml

@@ -5,41 +5,30 @@ tasks:
     class: lsst.analysis.tools.tasks.ObjectTableTractAnalysisTask
     config:
       connections.outputName: objectTableCore
-      plots.shapeSizeFractionalDiffScatterPlot: ShapeSizeFractionalDiffScatterPlot
-      metrics.shapeSizeFractionalMetric: ShapeSizeFractionalMetric
-      plots.e1DiffScatterPlot: E1DiffScatterPlot
-      metrics.e1DiffScatterMetric: E1DiffMetric
-      plots.e2DiffScatterPlot: E2DiffScatterPlot
-      metrics.e2DiffScatterMetric: E2DiffMetric
-      metrics.skyFluxStatisticMetric: SkyFluxStatisticMetric
-      metrics.skyFluxStatisticMetric.applyContext: CoaddContext
-      plots.wPerpPSFPlot: WPerpPSFPlot
-      metrics.wPerpPSFMetric: WPerpPSFMetric
-      plots.wPerpCModelPlot: WPerpCModelPlot
-      metrics.wPerpCModelMetric: WPerpCModelMetric
-      plots.xPerpPSFPlot: XPerpPSFPlot
-      metrics.xPerpPSFMetric: XPerpPSFMetric
-      plots.xPerpCModelPlot: XPerpCModelPlot
-      metrics.xPerpCModelMetric: XPerpCModelMetric
-      plots.yPerpPSFPlot: YPerpPSFPlot
-      metrics.yPerpPSFMetric: YPerpPSFMetric
-      plots.yPerpCModelPlot: YPerpCModelPlot
-      metrics.yPerpCModelMetric: YPerpCModelMetric
-      plots.skyObjectSkyPlot: SkyObjectSkyPlot
-      plots.skyObjectHistPlot: SkyObjectHistPlot
+      atools.shapeSizeFractionalDiff: ShapeSizeFractionalDiff
+      atools.e1Diff: E1Diff
+      atools.e2Diff: E2Diff
+      atools.skyFluxStatisticMetric: SkyFluxStatisticMetric
+      atools.skyFluxStatisticMetric.applyContext: CoaddContext
+      atools.wPerpPSFP: WPerpPSF
+      atools.wPerpCModel: WPerpCModel
+      atools.xPerpPSFP: XPerpPSF
+      atools.xPerpCModel: XPerpCModel
+      atools.yPerpPSF: YPerpPSF
+      atools.yPerpCModel: YPerpCModel
+      atools.skyObjectSkyPlot: SkyObjectSkyPlot
+      atools.skyObjectHistPlot: SkyObjectHistPlot
       python: |
-        from lsst.analysis.tools.analysisPlots import *
-        from lsst.analysis.tools.analysisMetrics import *
+        from lsst.analysis.tools.atools import *
         from lsst.analysis.tools.contexts import *
   analyzeObjectTableSurveyCore:
     class: lsst.analysis.tools.tasks.ObjectTableSurveyAnalysisTask
     config:
       connections.outputName: objectTableSurvey
-      plots.rhoStatisticsPlot: RhoStatisticsPlot
+      plots.rhoStatistics: RhoStatistics
       bands: ["i"]
       python: |
-        from lsst.analysis.tools.analysisPlots import *
-        from lsst.analysis.tools.analysisMetrics import *
+        from lsst.analysis.tools.atools import *
         from lsst.analysis.tools.contexts import *
         from lsst.analysis.tools.actions.plot import *
   catalogMatchTract:

pipelines/coaddQualityExtended.yaml

@@ -6,7 +6,6 @@ tasks:
     config:
       connections.outputName: objectTableExtended
       # set plots to run
-      plots.ap12PsfSkyPlot: Ap12PsfSkyPlot
+      atools.ap12PsfSkyPlot: Ap12PsfSkyPlot
       python: |
-        from lsst.analysis.tools.analysisPlots import *
-        from lsst.analysis.tools.analysisMetrics import *
+        from lsst.analysis.tools.atools import *

pipelines/diaObjectAssociatedSources.yaml

@@ -6,8 +6,6 @@ tasks:
     config:
       connections.outputName: sourcesOutput
       bands: ['g']
-      metrics.sourceMetrics: CountSources
-      plots.analysisPlots: SourcesPlot
+      atools.sourceTool: SourcesTool
       python: |
-        from lsst.analysis.tools.analysisMetrics import *
-        from lsst.analysis.tools.analysisPlots import *
+        from lsst.analysis.tools.atools import *

pipelines/matchedVisitQualityCore.yaml

@@ -6,8 +6,6 @@ tasks:
     config:
       bands: ['g', 'r', 'i', 'z']
       connections.outputName: matchedVisitCore
-      metrics.stellarPhotometricRepeatabilityMetric: StellarPhotometricRepeatabilityMetric
-      plots.StellarPhotometricRepeatabilityHistPlot: StellarPhotometricRepeatabilityHistPlot
+      atools.stellarPhotometricRepeatabilityMetric: StellarPhotometricRepeatability
       python: |
-        from lsst.analysis.tools.analysisMetrics import *
-        from lsst.analysis.tools.analysisPlots import *
+        from lsst.analysis.tools.atools import *

pipelines/visitColumnValidate.yaml

@@ -5,42 +5,42 @@ tasks:
     class: lsst.analysis.tools.tasks.SourceTableVisitAnalysisTask
     config:
       connections.outputName: visitTableColumnValidate
-      metrics.validPsfFluxMetric: ValidFracColumnMetric
-      metrics.validPsfFluxMetric.vectorKey: 'psfFlux'
-      metrics.validPsfFluxMetric.applyContext: VisitContext
-      metrics.validAp03FluxMetric: ValidFracColumnMetric
-      metrics.validAp03FluxMetric.vectorKey: 'ap03Flux'
-      metrics.validAp03FluxMetric.applyContext: VisitContext
-      metrics.validAp06FluxMetric: ValidFracColumnMetric
-      metrics.validAp06FluxMetric.vectorKey: 'ap06Flux'
-      metrics.validAp06FluxMetric.applyContext: VisitContext
-      metrics.validAp09FluxMetric: ValidFracColumnMetric
-      metrics.validAp09FluxMetric.vectorKey: 'ap09Flux'
-      metrics.validAp09FluxMetric.applyContext: VisitContext
-      metrics.validAp12FluxMetric: ValidFracColumnMetric
-      metrics.validAp12FluxMetric.vectorKey: 'ap12Flux'
-      metrics.validAp12FluxMetric.applyContext: VisitContext
-      metrics.validAp17FluxMetric: ValidFracColumnMetric
-      metrics.validAp17FluxMetric.vectorKey: 'ap17Flux'
-      metrics.validAp17FluxMetric.applyContext: VisitContext
-      metrics.validAp25FluxMetric: ValidFracColumnMetric
-      metrics.validAp25FluxMetric.vectorKey: 'ap25Flux'
-      metrics.validAp25FluxMetric.applyContext: VisitContext
-      metrics.validAp35FluxMetric: ValidFracColumnMetric
-      metrics.validAp35FluxMetric.vectorKey: 'ap35Flux'
-      metrics.validAp35FluxMetric.applyContext: VisitContext
-      metrics.validAp50FluxMetric: ValidFracColumnMetric
-      metrics.validAp50FluxMetric.vectorKey: 'ap50Flux'
-      metrics.validAp50FluxMetric.applyContext: VisitContext
-      metrics.validAp70FluxMetric: ValidFracColumnMetric
-      metrics.validAp70FluxMetric.vectorKey: 'ap70Flux'
-      metrics.validAp70FluxMetric.applyContext: VisitContext
-      metrics.validCalibFluxMetric: ValidFracColumnMetric
-      metrics.validCalibFluxMetric.vectorKey: 'calibFlux'
-      metrics.validCalibFluxMetric.applyContext: VisitContext
-      metrics.validGaussianFluxMetric: ValidFracColumnMetric
-      metrics.validGaussianFluxMetric.vectorKey: 'gaussianFlux'
-      metrics.validGaussianFluxMetric.applyContext: VisitContext
+      atools.validPsfFluxMetric: ValidFracColumnMetric
+      atools.validPsfFluxMetric.vectorKey: 'psfFlux'
+      atools.validPsfFluxMetric.applyContext: VisitContext
+      atools.validAp03FluxMetric: ValidFracColumnMetric
+      atools.validAp03FluxMetric.vectorKey: 'ap03Flux'
+      atools.validAp03FluxMetric.applyContext: VisitContext
+      atools.validAp06FluxMetric: ValidFracColumnMetric
+      atools.validAp06FluxMetric.vectorKey: 'ap06Flux'
+      atools.validAp06FluxMetric.applyContext: VisitContext
+      atools.validAp09FluxMetric: ValidFracColumnMetric
+      atools.validAp09FluxMetric.vectorKey: 'ap09Flux'
+      atools.validAp09FluxMetric.applyContext: VisitContext
+      atools.validAp12FluxMetric: ValidFracColumnMetric
+      atools.validAp12FluxMetric.vectorKey: 'ap12Flux'
+      atools.validAp12FluxMetric.applyContext: VisitContext
+      atools.validAp17FluxMetric: ValidFracColumnMetric
+      atools.validAp17FluxMetric.vectorKey: 'ap17Flux'
+      atools.validAp17FluxMetric.applyContext: VisitContext
+      atools.validAp25FluxMetric: ValidFracColumnMetric
+      atools.validAp25FluxMetric.vectorKey: 'ap25Flux'
+      atools.validAp25FluxMetric.applyContext: VisitContext
+      atools.validAp35FluxMetric: ValidFracColumnMetric
+      atools.validAp35FluxMetric.vectorKey: 'ap35Flux'
+      atools.validAp35FluxMetric.applyContext: VisitContext
+      atools.validAp50FluxMetric: ValidFracColumnMetric
+      atools.validAp50FluxMetric.vectorKey: 'ap50Flux'
+      atools.validAp50FluxMetric.applyContext: VisitContext
+      atools.validAp70FluxMetric: ValidFracColumnMetric
+      atools.validAp70FluxMetric.vectorKey: 'ap70Flux'
+      atools.validAp70FluxMetric.applyContext: VisitContext
+      atools.validCalibFluxMetric: ValidFracColumnMetric
+      atools.validCalibFluxMetric.vectorKey: 'calibFlux'
+      atools.validCalibFluxMetric.applyContext: VisitContext
+      atools.validGaussianFluxMetric: ValidFracColumnMetric
+      atools.validGaussianFluxMetric.vectorKey: 'gaussianFlux'
+      atools.validGaussianFluxMetric.applyContext: VisitContext
       python: |
-        from lsst.analysis.tools.analysisMetrics import *
+        from lsst.analysis.tools.atools import *
         from lsst.analysis.tools.contexts import *

pipelines/visitQualityCore.yaml

@@ -8,11 +8,10 @@ tasks:
       # plots.label = APlotClass
       # metrics.name = AMetricsClass
       connections.outputName: sourceTableCore
-      metrics.skyFluxVisitStatisticMetric: SkyFluxStatisticMetric
-      metrics.skyFluxVisitStatisticMetric.applyContext: VisitContext
-      plots.skySourceSkyPlot: SkySourceSkyPlot
-      plots.skySourceHistPlot: SkySourceHistPlot
+      atools.skyFluxVisitStatisticMetric: SkyFluxStatisticMetric
+      atools.skyFluxVisitStatisticMetric.applyContext: VisitContext
+      atools.skySourceSkyPlot: SkySourceSkyPlot
+      atools.skySourceHistPlot: SkySourceHistPlot
       python: |
-        from lsst.analysis.tools.analysisPlots import *
-        from lsst.analysis.tools.analysisMetrics import *
+        from lsst.analysis.tools.atools import *
         from lsst.analysis.tools.contexts import *

python/lsst/__init__.py

@@ -0,0 +1,3 @@
+import pkgutil
+
+__path__ = pkgutil.extend_path(__path__, __name__)

python/lsst/analysis/__init__.py

@@ -0,0 +1,3 @@
+import pkgutil
+
+__path__ = pkgutil.extend_path(__path__, __name__)

python/lsst/analysis/tools/actions/plot/scatterplotWithTwoHists.py

@@ -39,7 +39,7 @@
 from matplotlib.path import Path
 from mpl_toolkits.axes_grid1 import make_axes_locatable
 
-from ...interfaces import KeyedData, KeyedDataAction, KeyedDataSchema, PlotAction, Scalar, Vector
+from ...interfaces import KeyedData, KeyedDataAction, KeyedDataSchema, PlotAction, Scalar, ScalarType, Vector
 from ...statistics import nansigmaMad, sigmaMad
 from ..keyedData.summaryStatistics import SummaryStatisticAction
 from ..scalar import MedianAction
@@ -167,7 +167,7 @@ class ScatterPlotWithTwoHists(PlotAction):
     _stats = ("median", "sigmaMad", "count", "approxMag")
 
     def getInputSchema(self) -> KeyedDataSchema:
-        base: list[tuple[str, type[Vector] | type[Scalar]]] = []
+        base: list[tuple[str, type[Vector] | ScalarType]] = []
         if "stars" in self.plotTypes:  # type: ignore
             base.append(("xStars", Vector))
             base.append(("yStars", Vector))

python/lsst/analysis/tools/actions/vector/ellipticity.py

@@ -27,6 +27,8 @@
     "CalcE2",
 )
 
+from typing import cast
+
 import numpy as np
 from lsst.pex.config import ChoiceField, Field, FieldValidationError
 from lsst.pex.config.configurableActions import ConfigurableActionField
@@ -126,6 +128,8 @@ def __call__(self, data: KeyedData, **kwargs) -> Vector:
                 data[self.colXx.format(**kwargs)] * data[self.colYy.format(**kwargs)]
                 - data[self.colXy.format(**kwargs)] ** 2
             )
+        e = cast(Vector, e)
+        denom = cast(Vector, denom)
 
         e /= denom
 
@@ -281,7 +285,7 @@ def __call__(self, data: KeyedData, **kwargs) -> Vector:
             )
         e1 = (data[self.colXx.format(**kwargs)] - data[self.colYy.format(**kwargs)]) / denom
 
-        return e1
+        return cast(Vector, e1)
 
     def validate(self):
         super().validate()
@@ -344,4 +348,4 @@ def __call__(self, data: KeyedData, **kwargs) -> Vector:
                 - data[self.colXy.format(**kwargs)] ** 2
             )
         e2 = 2 * data[self.colXy.format(**kwargs)] / denom
-        return e2
+        return cast(Vector, e2)