Merge pull request #154 from lsst/tickets/DM-41118

DM-41118: Add metrics from diffim task metadata

pipelines/apDetectorVisitQualityCore.yaml

@@ -20,3 +20,26 @@ tasks:
       connections.outputName: trailedDiaSrcCore
       python: |
         from lsst.analysis.tools.atools import *
+  diffimTaskCore:
+    class: lsst.analysis.tools.tasks.DiffimDetectorVisitAnalysisTask
+    config:
+      connections.outputName: diffimMetadata
+
+      atools.diffimMetadataMetric: DiffimMetadataMetricTool
+      atools.diffimMetadataMetric.metrics:
+        # Format is "metric name in the metadata": units
+        nUnmergedDiaSources: ct
+        nMergedDiaSources: ct
+        nGoodPixels: ct
+        nBadPixels: ct
+        nPixelsDetectedPositive: ct
+        nPixelsDetectedNegative: ct
+        nBadPixelsDetectedPositive: ct
+        nBadPixelsDetectedNegative: ct
+        sciencePsfSize: pixel
+        templatePsfSize: pixel
+        scaleScienceVarianceFactor: ct
+        scaleTemplateVarianceFactor: ct
+        templateCoveragePercent: percent
+      python: |
+        from lsst.analysis.tools.atools import DiffimMetadataMetricTool

python/lsst/analysis/tools/actions/scalar/scalarActions.py

@@ -4,6 +4,7 @@
     "MedianAction",
     "MeanAction",
     "StdevAction",
+    "ValueAction",
     "SigmaMadAction",
     "CountAction",
     "CountUniqueAction",
@@ -64,6 +65,18 @@ def __call__(self, data: KeyedData, **kwargs) -> Scalar:
         return cast(Scalar, float(np.nanstd(cast(Vector, data[self.vectorKey.format(**kwargs)])[mask])))
 
 
+class ValueAction(ScalarAction):
+    """Extracts the first value from a vector."""
+
+    vectorKey = Field[str]("Key of Vector from which to extract the first value")
+
+    def getInputSchema(self) -> KeyedDataSchema:
+        return ((self.vectorKey, Vector),)
+
+    def __call__(self, data: KeyedData, **kwargs) -> Scalar:
+        return cast(Scalar, float(data[self.vectorKey.format(**kwargs)][0]))
+
+
 class SigmaMadAction(ScalarAction):
     """Calculates the sigma mad of the given data."""
 

python/lsst/analysis/tools/atools/__init__.py

@@ -3,6 +3,7 @@
 from .coveragePlots import *
 from .diaSolarSystemObjectMetrics import *
 from .diaSourceMetrics import *
+from .diffimMetadataMetrics import *
 from .diffMatched import *
 from .fluxMetrics import *
 from .genericBuild import *

python/lsst/analysis/tools/atools/diffimMetadataMetrics.py

@@ -0,0 +1,39 @@
+# This file is part of analysis_tools.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+__all__ = ("DiffimMetadataMetricTool",)
+
+from lsst.pex.config import DictField
+
+from ..actions.scalar import ValueAction
+from ..interfaces import AnalysisTool
+
+
+class DiffimMetadataMetricTool(AnalysisTool):
+    """This tool is designed to extract values from diffim task metadata"""
+
+    metrics = DictField[str, str](doc="The metrics to calculate from the task metadata and their units")
+
+    def finalize(self):
+        for name, unit in self.metrics.items():
+            setattr(self.process.calculateActions, name, ValueAction(vectorKey=name))
+        self.produce.metric.units = dict(self.metrics.items())

python/lsst/analysis/tools/tasks/__init__.py

@@ -3,6 +3,7 @@
 from .catalogMatch import *
 from .ccdVisitTableAnalysis import *
 from .diaObjectDetectorVisitAnalysis import *
+from .diffimTaskDetectorVisitAnalysis import *
 from .diffMatchedAnalysis import *
 from .gatherResourceUsage import *
 from .objectTableSurveyAnalysis import *

python/lsst/analysis/tools/tasks/assocDiaSrcDetectorVisitAnalysis.py

@@ -22,17 +22,17 @@
 
 __all__ = ("AssocDiaSrcDetectorVisitAnalysisConfig", "AssocDiaSrcDetectorVisitAnalysisTask")
 
-from lsst.pipe.base import connectionTypes as ct
+from lsst.pipe.base import connectionTypes
 
 from ..interfaces import AnalysisBaseConfig, AnalysisBaseConnections, AnalysisPipelineTask
 
 
 class AssocDiaSrcDetectorVisitAnalysisConnections(
     AnalysisBaseConnections,
     dimensions=("visit", "band", "detector"),
-    defaultTemplates={"coaddName": "goodSeeing", "fakesType": "fakes_"},
+    defaultTemplates={"coaddName": "goodSeeing", "fakesType": ""},
 ):
-    data = ct.Input(
+    data = connectionTypes.Input(
         doc="CcdVisit-based DiaSource table to load from the butler",
         name="{fakesType}{coaddName}Diff_assocDiaSrc",
         storageClass="DataFrame",

python/lsst/analysis/tools/tasks/diffimTaskDetectorVisitAnalysis.py

@@ -0,0 +1,69 @@
+# This file is part of analysis_tools.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+__all__ = ("DiffimDetectorVisitAnalysisConfig", "DiffimDetectorVisitAnalysisTask")
+
+import pandas as pd
+from lsst.pipe.base import connectionTypes
+
+from ..interfaces import AnalysisBaseConfig, AnalysisBaseConnections, AnalysisPipelineTask
+
+
+class DiffimDetectorVisitAnalysisConnections(
+    AnalysisBaseConnections,
+    dimensions=("visit", "band", "detector"),
+):
+    metadataSubtract = connectionTypes.Input(
+        doc="Task metadata from image differencing",
+        name="subtractImages_metadata",
+        storageClass="TaskMetadata",
+        dimensions=("visit", "band", "detector"),
+    )
+    metadataDetect = connectionTypes.Input(
+        doc="Task metadata from DIA detection and measurement",
+        name="detectAndMeasure_metadata",
+        storageClass="TaskMetadata",
+        dimensions=("visit", "band", "detector"),
+    )
+
+
+class DiffimDetectorVisitAnalysisConfig(
+    AnalysisBaseConfig, pipelineConnections=DiffimDetectorVisitAnalysisConnections
+):
+    pass
+
+
+class DiffimDetectorVisitAnalysisTask(AnalysisPipelineTask):
+    ConfigClass = DiffimDetectorVisitAnalysisConfig
+    _DefaultName = "DiffimDetectorVisitAnalysis"
+
+    def runQuantum(self, butlerQC, inputRefs, outputRefs):
+        inputs = butlerQC.get(inputRefs)
+        metadata = inputs["metadataDetect"].metadata["detectAndMeasure"].to_dict()
+        inputs.pop("metadataDetect")
+        metadata |= inputs["metadataSubtract"].metadata["subtractImages"].to_dict()
+        inputs.pop("metadataSubtract")
+        df = pd.DataFrame(metadata)
+
+        inputs["data"] = df
+        outputs = self.run(**inputs)
+        butlerQC.put(outputs, outputRefs)