Add metrics from detectAndMeasure metadata

pipelines/apDetectorVisitQualityCore.yaml

@@ -12,3 +12,22 @@ tasks:
       atools.simpleSky: SimpleDiaPlot
       python: |
         from lsst.analysis.tools.atools import *
+  diffimTaskCore:
+    class: lsst.analysis.tools.tasks.DiffimDetectorVisitAnalysisTask
+    config:
+      connections.outputName: diffimMetadata
+      atools.nUnmergedDiaSources: NUnmergedDiaSourcesMetric
+      atools.nMergedDiaSources: NMergedDiaSourcesMetric
+      atools.nGoodPixelsMetric: NGoodPixelsMetric
+      atools.nBadPixelsMetric: NBadPixelsMetric
+      atools.nDetectedPositiveAllPixelsMetric: NPixelsDetectedPositiveMetric
+      atools.nDetectedNegativeAllPixelsMetric: NPixelsDetectedNegativeMetric
+      atools.nDetectedPositiveGoodPixelsMetric: NBadPixelsDetectedPositiveMetric
+      atools.nDetectedNegativeGoodPixelsMetric: NBadPixelsDetectedNegativeMetric
+      atools.sciencePsfSize: SciencePsfSizeMetric
+      atools.templatePsfSize: TemplatePsfSizeMetric
+      atools.scaleScienceVariance: ScienceVarianceScaleMetric
+      atools.scaleTemplateVariance: TemplateVarianceScaleMetric
+      atools.templateCoverage: TemplateCoverageMetric
+      python: |
+        from lsst.analysis.tools.atools import *

python/lsst/analysis/tools/atools/__init__.py

@@ -3,6 +3,7 @@
 from .coveragePlots import *
 from .diaSolarSystemObjectMetrics import *
 from .diaSourceMetrics import *
+from .diffimMetadataMetrics import *
 from .diffMatched import *
 from .fluxMetrics import *
 from .genericBuild import *

python/lsst/analysis/tools/atools/diffimMetadataMetrics.py

@@ -0,0 +1,225 @@
+# This file is part of analysis_tools.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+__all__ = (
+    "NUnmergedDiaSourcesMetric",
+    "NMergedDiaSourcesMetric",
+    "NGoodPixelsMetric",
+    "NBadPixelsMetric",
+    "NPixelsDetectedPositiveMetric",
+    "NPixelsDetectedNegativeMetric",
+    "NBadPixelsDetectedPositiveMetric",
+    "NBadPixelsDetectedNegativeMetric",
+    "SciencePsfSizeMetric",
+    "TemplatePsfSizeMetric",
+    "ScienceVarianceScaleMetric",
+    "TemplateVarianceScaleMetric",
+    "TemplateCoverageMetric",
+)
+
+from ..actions.scalar import MedianAction
+from ..interfaces import AnalysisTool
+
+
+class NUnmergedDiaSourcesMetric(AnalysisTool):
+    """The raw number of DIA source footprints."""
+
+    def setDefaults(self):
+        super().setDefaults()
+
+        # Count the number of dia sources
+        self.process.calculateActions.nUnmergedDiaSources = MedianAction(
+            vectorKey="nUnmergedDiaSources"
+        )
+
+        # the units for the quantity (count, an astropy quantity)
+        self.produce.metric.units = {"nUnmergedDiaSources": "ct"}
+
+
+class NMergedDiaSourcesMetric(AnalysisTool):
+    """The number of DIA Sources."""
+
+    def setDefaults(self):
+        super().setDefaults()
+
+        # Count the number of dia sources
+        self.process.calculateActions.nMergedDiaSources = MedianAction(
+            vectorKey="nMergedDiaSources"
+        )
+
+        # the units for the quantity (count, an astropy quantity)
+        self.produce.metric.units = {"nMergedDiaSources": "ct"}
+
+
+class NGoodPixelsMetric(AnalysisTool):
+    """The number of unflagged pixels in the difference image."""
+
+    def setDefaults(self):
+        super().setDefaults()
+
+        # Count the number of dia sources
+        self.process.calculateActions.nGoodPixels = MedianAction(vectorKey="nGoodPixels")
+
+        # the units for the quantity (count, an astropy quantity)
+        self.produce.metric.units = {"nGoodPixels": "ct"}
+
+
+class NBadPixelsMetric(AnalysisTool):
+    """The number of flagged pixels in the difference image."""
+
+    def setDefaults(self):
+        super().setDefaults()
+
+        # Count the number of dia sources
+        self.process.calculateActions.nBadPixels = MedianAction(vectorKey="nBadPixels")
+
+        # the units for the quantity (count, an astropy quantity)
+        self.produce.metric.units = {"nBadPixels": "ct"}
+
+
+class NPixelsDetectedPositiveMetric(AnalysisTool):
+    """The number of pixels in the footprints of positive sources."""
+
+    def setDefaults(self):
+        super().setDefaults()
+
+        # Count the number of dia sources
+        self.process.calculateActions.nPixelsDetectedPositive = MedianAction(
+            vectorKey="nPixelsDetectedPositive"
+        )
+
+        # the units for the quantity (count, an astropy quantity)
+        self.produce.metric.units = {"nPixelsDetectedPositive": "ct"}
+
+
+class NPixelsDetectedNegativeMetric(AnalysisTool):
+    """The number of pixels in the footprints of negative sources."""
+
+    def setDefaults(self):
+        super().setDefaults()
+
+        # Count the number of dia sources
+        self.process.calculateActions.nPixelsDetectedNegative = MedianAction(
+            vectorKey="nPixelsDetectedNegative"
+        )
+
+        # the units for the quantity (count, an astropy quantity)
+        self.produce.metric.units = {"nPixelsDetectedNegative": "ct"}
+
+
+class NBadPixelsDetectedPositiveMetric(AnalysisTool):
+    """The number of flagged pixels in the footprints of positive sources."""
+
+    def setDefaults(self):
+        super().setDefaults()
+
+        # Count the number of dia sources
+        self.process.calculateActions.nBadPixelsDetectedPositive = MedianAction(
+            vectorKey="nBadPixelsDetectedPositive"
+        )
+
+        # the units for the quantity (count, an astropy quantity)
+        self.produce.metric.units = {"nBadPixelsDetectedPositive": "ct"}
+
+
+class NBadPixelsDetectedNegativeMetric(AnalysisTool):
+    """The number of flagged pixels in the footprints of negative sources."""
+
+    def setDefaults(self):
+        super().setDefaults()
+
+        # Count the number of dia sources
+        self.process.calculateActions.nBadPixelsDetectedNegative = MedianAction(
+            vectorKey="nBadPixelsDetectedNegative"
+        )
+
+        # the units for the quantity (count, an astropy quantity)
+        self.produce.metric.units = {"nBadPixelsDetectedNegative": "ct"}
+
+
+class SciencePsfSizeMetric(AnalysisTool):
+    """PSF size of the science image."""
+
+    def setDefaults(self):
+        super().setDefaults()
+
+        # Count the number of dia sources
+        self.process.calculateActions.sciencePsfSize = MedianAction(vectorKey="sciencePsfSize")
+
+        # the units for the quantity
+        self.produce.metric.units = {"sciencePsfSize": "pixel"}
+
+
+class TemplatePsfSizeMetric(AnalysisTool):
+    """PSF size of the template image."""
+
+    def setDefaults(self):
+        super().setDefaults()
+
+        # Count the number of dia sources
+        self.process.calculateActions.templatePsfSize = MedianAction(vectorKey="templatePsfSize")
+
+        # the units for the quantity
+        self.produce.metric.units = {"templatePsfSize": "pixel"}
+
+
+class ScienceVarianceScaleMetric(AnalysisTool):
+    """Factor from ScaleVarianceTask for the science image."""
+
+    def setDefaults(self):
+        super().setDefaults()
+
+        # Count the number of dia sources
+        self.process.calculateActions.scaleScienceVariance = MedianAction(
+            vectorKey="scaleScienceVarianceFactor"
+        )
+
+        # the units for the quantity
+        self.produce.metric.units = {"scaleScienceVariance": "pixel"}
+
+
+class TemplateVarianceScaleMetric(AnalysisTool):
+    """Factor from ScaleVarianceTask for the template image."""
+
+    def setDefaults(self):
+        super().setDefaults()
+
+        # Count the number of dia sources
+        self.process.calculateActions.scaleTemplateVariance = MedianAction(
+            vectorKey="scaleTemplateVarianceFactor"
+        )
+
+        # the units for the quantity
+        self.produce.metric.units = {"scaleTemplateVariance": "pixel"}
+
+
+class TemplateCoverageMetric(AnalysisTool):
+    """Percent of pixels with data in the template image."""
+
+    def setDefaults(self):
+        super().setDefaults()
+
+        # Count the number of dia sources
+        self.process.calculateActions.templateCoverage = MedianAction(vectorKey="TemplateCoveragePercent")
+
+        # the units for the quantity
+        self.produce.metric.units = {"templateCoverage": "percent"}

python/lsst/analysis/tools/tasks/__init__.py

@@ -3,6 +3,7 @@
 from .catalogMatch import *
 from .ccdVisitTableAnalysis import *
 from .diaObjectDetectorVisitAnalysis import *
+from .diffimTaskDetectorVisitAnalysis import *
 from .diffMatchedAnalysis import *
 from .objectTableSurveyAnalysis import *
 from .objectTableTractAnalysis import *

python/lsst/analysis/tools/tasks/diffimTaskDetectorVisitAnalysis.py

@@ -0,0 +1,70 @@
+# This file is part of analysis_tools.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+__all__ = ("DiffimDetectorVisitAnalysisConfig", "DiffimDetectorVisitAnalysisTask")
+
+import pandas as pd
+from lsst.pipe.base import connectionTypes as ct
+
+from ..interfaces import AnalysisBaseConfig, AnalysisBaseConnections, AnalysisPipelineTask
+
+
+class DiffimDetectorVisitAnalysisConnections(
+    AnalysisBaseConnections,
+    dimensions=("visit", "band", "detector"),
+    defaultTemplates={"coaddName": "goodSeeing", "fakesType": "fakes_"},
+):
+    metadataSubtract = ct.Input(
+        doc="Task metadata from image differencing",
+        name="subtractImages_metadata",
+        storageClass="TaskMetadata",
+        dimensions=("visit", "band", "detector"),
+    )
+    metadataDetect = ct.Input(
+        doc="Task metadata from DIA detection and measurement",
+        name="detectAndMeasure_metadata",
+        storageClass="TaskMetadata",
+        dimensions=("visit", "band", "detector"),
+    )
+
+
+class DiffimDetectorVisitAnalysisConfig(
+    AnalysisBaseConfig, pipelineConnections=DiffimDetectorVisitAnalysisConnections
+):
+    pass
+
+
+class DiffimDetectorVisitAnalysisTask(AnalysisPipelineTask):
+    ConfigClass = DiffimDetectorVisitAnalysisConfig
+    _DefaultName = "DiffimDetectorVisitAnalysis"
+
+    def runQuantum(self, butlerQC, inputRefs, outputRefs):
+        inputs = butlerQC.get(inputRefs)
+        metadata = inputs["metadataDetect"].metadata["detectAndMeasure"].to_dict()
+        inputs.pop("metadataDetect")
+        metadata |= inputs["metadataSubtract"].metadata["subtractImages"].to_dict()
+        inputs.pop("metadataSubtract")
+        df = pd.DataFrame(metadata)
+
+        inputs["data"] = df
+        outputs = self.run(**inputs)
+        butlerQC.put(outputs, outputRefs)