DM-27388: Implement metric system for fakes in AP

python/lsst/ap/pipe/metrics.py

@@ -0,0 +1,141 @@
+# This file is part of ap_pipe.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""Metrics for ap_pipe tasks.
+"""
+
+__all__ = [
+    "ApFakesCompletenessMetricTask", "ApFakesCompletenessMetricConfig",
+]
+
+import astropy.units as u
+import numpy as np
+import traceback
+
+import lsst.pex.config as pexConfig
+from lsst.pipe.base import Struct
+import lsst.pipe.base.connectionTypes as connTypes
+from lsst.pipe.tasks.insertFakes import InsertFakesConfig
+from lsst.verify import Measurement
+from lsst.verify.tasks import MetricTask, MetricComputationError
+
+
+class ApFakesCompletenessMetricConnections(
+        MetricTask.ConfigClass.ConnectionsClass,
+        dimensions={"instrument", "visit", "detector", "band"},
+        defaultTemplates={"coaddName": "deep",
+                          "fakesType": "",
+                          "package": "ap_pipe",
+                          "metric": "apFakesCompleteness"}):
+    """ApFakesCompleteness connections.
+    """
+    matchedFakes = connTypes.Input(
+        doc="Fakes matched to their detections in the difference image.",
+        name="{fakesType}{coaddName}Diff_matchDiaSrc",
+        storageClass="DataFrame",
+        dimensions=("instrument", "visit", "detector"),
+    )
+
+
+# Inherits from InsertFakesConfig to preserve column names in the fakes
+# catalog.
+class ApFakesCompletenessMetricConfig(
+        MetricTask.ConfigClass,
+        InsertFakesConfig,
+        pipelineConnections=ApFakesCompletenessMetricConnections):
+    """ApFakesCompleteness config.
+    """
+    magMin = pexConfig.RangeField(
+        doc="Minimum of cut on magnitude range used to compute completeness "
+            "in.",
+        dtype=float,
+        default=20,
+        min=1,
+        max=40,
+    )
+    magMax = pexConfig.RangeField(
+        doc="Maximum of cut on magnitude range used to compute completeness "
+            "in.",
+        dtype=int,
+        default=30,
+        min=1,
+        max=40,
+    )
+
+
+class ApFakesCompletenessMetricTask(MetricTask):
+    """Metric task for summarizing the completeness of fakes inserted into the
+    AP pipeline.
+    """
+    _DefaultName = "apFakesCompleteness"
+    ConfigClass = ApFakesCompletenessMetricConfig
+
+    def runQuantum(self, butlerQC, inputRefs, outputRefs):
+        try:
+            inputs = butlerQC.get(inputRefs)
+            inputs["band"] = butlerQC.quantum.dataId["band"]
+            outputs = self.run(**inputs)
+            if outputs.measurement is not None:
+                butlerQC.put(outputs, outputRefs)
+            else:
+                self.log.debugf("Skipping measurement of {!r} on {} "
+                                "as not applicable.", self, inputRefs)
+        except MetricComputationError:
+            # Apparently lsst.log doesn't have built-in exception support?
+            self.log.errorf(
+                "Measurement of {!r} failed on {}->{}\n{}",
+                self, inputRefs, outputRefs, traceback.format_exc())
+
+    def run(self, matchedFakes, band):
+        """Compute the completeness of recovered fakes within a magnitude
+        range.
+
+        Parameters
+        ----------
+        matchedFakes : `lsst.afw.table.SourceCatalog` or `None`
+            Catalog of fakes that were inserted into the ccdExposure matched
+            to their detected counterparts.
+
+        Returns
+        -------
+        result : `lsst.pipe.base.Struct`
+            A `~lsst.pipe.base.Struct` containing the following component:
+            ``measurement``
+                the ratio (`lsst.verify.Measurement` or `None`)
+        """
+        if matchedFakes is not None:
+            magnitudes = matchedFakes[f"{self.config.magVar}" % band]
+            magCutFakes = matchedFakes[np.logical_and(magnitudes > self.config.magMin,
+                                                      magnitudes < self.config.magMax)]
+            if len(magCutFakes) <= 0.0:
+                raise MetricComputationError(
+                    "No matched fakes catalog sources found; Completeness is "
+                    "ill defined.")
+            else:
+                meas = Measurement(
+                    self.config.metricName,
+                    ((magCutFakes["diaSourceId"] > 0).sum() / len(magCutFakes))
+                    * u.dimensionless_unscaled)
+        else:
+            self.log.info("Nothing to do: no matched catalog found.")
+            meas = None
+        return Struct(measurement=meas)

tests/test_metrics.py

@@ -0,0 +1,136 @@
+# This file is part of ap_pipe.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import astropy.units as u
+import numpy as np
+import unittest
+
+from lsst.pipe.base import testUtils
+import lsst.skymap as skyMap
+import lsst.utils.tests
+from lsst.verify import Name
+from lsst.verify.tasks.testUtils import MetricTaskTestCase
+
+from lsst.ap.pipe.createApFakes import CreateRandomApFakesTask, CreateRandomApFakesConfig
+from lsst.ap.pipe.metrics import ApFakesCompletenessMetricTask, ApFakesCompletenessMetricConfig
+
+
+class TestApCompletenessTask(MetricTaskTestCase):
+
+    @classmethod
+    def makeTask(cls, magMin=20, magMax=30):
+        """Make the task and allow for modification of the config min and max.
+
+        Parameters
+        ----------
+        magMin : min magnitude, `float`
+            Minimum magnitude
+        magMax : min magnitude, `float`
+            Maximum magnitude
+        """
+        config = ApFakesCompletenessMetricConfig()
+        config.magMin = magMin
+        config.magMax = magMax
+
+        return ApFakesCompletenessMetricTask(config=config)
+
+    def setUp(self):
+        super().setUp()
+
+        simpleMapConfig = skyMap.discreteSkyMap.DiscreteSkyMapConfig()
+        simpleMapConfig.raList = [45]
+        simpleMapConfig.decList = [45]
+        simpleMapConfig.radiusList = [0.1]
+
+        self.simpleMap = skyMap.DiscreteSkyMap(simpleMapConfig)
+        self.tractId = 0
+        bCircle = self.simpleMap.generateTract(self.tractId).getInnerSkyPolygon().getBoundingCircle()
+        self.targetSources = 1000
+        self.sourceDensity = (self.targetSources
+                              / (bCircle.getArea() * (180 / np.pi) ** 2))
+
+        fakesConfig = CreateRandomApFakesConfig()
+        fakesConfig.fraction = 0.0
+        fakesConfig.fakeDensity = self.sourceDensity
+        fakesTask = CreateRandomApFakesTask(config=fakesConfig)
+        fakeCat = fakesTask.run(self.tractId, self.simpleMap).fakeCat
+
+        self.band = 'g'
+        self.magCut = 25
+        magMask = (fakeCat[fakesConfig.magVar % self.band] < self.magCut)
+        self.expectedAllMatched = magMask.sum()
+        ids = np.where(magMask, np.arange(1, len(fakeCat) + 1, dtype=int), 0)
+        # Add columns to mimic the matched fakes result without running the
+        # full pipeline.
+        self.fakeCat = fakeCat.assign(diaObjectId=ids,
+                                      filterName=["g"] * len(fakeCat),
+                                      diaSourceId=ids)
+
+    def testValid(self):
+        """Test the run method.
+        """
+        result = self.task.run(self.fakeCat, self.band)
+        testUtils.assertValidOutput(self.task, result)
+
+        meas = result.measurement
+        self.assertEqual(meas.metric_name, Name(metric="ap_pipe.apFakesCompleteness"))
+        # Work around for Mac failing this test.
+        self.assertAlmostEqual(
+            meas.quantity.value,
+            ((self.expectedAllMatched / self.targetSources) * u.dimensionless_unscaled).value,
+            places=2)
+
+    def testMissingData(self):
+        """Test the run method with no data.
+        """
+        result = self.task.run(None, None)
+        testUtils.assertValidOutput(self.task, result)
+        meas = result.measurement
+        self.assertIsNone(meas)
+
+    def testValidEmpty(self):
+        """Test the run method with a valid but zero result.
+        """
+        metricComplete = self.makeTask(self.magCut, self.magCut + 5)
+        result = metricComplete.run(self.fakeCat, self.band)
+        testUtils.assertValidOutput(metricComplete, result)
+
+        meas = result.measurement
+        self.assertEqual(meas.metric_name, Name(metric="ap_pipe.apFakesCompleteness"))
+        self.assertEqual(meas.quantity, 0 * u.dimensionless_unscaled)
+
+
+# Hack around unittest's hacky test setup system
+del MetricTaskTestCase
+
+
+class MemoryTester(lsst.utils.tests.MemoryTestCase):
+    pass
+
+
+def setup_module(module):
+    lsst.utils.tests.init()
+
+
+if __name__ == "__main__":
+    lsst.utils.tests.init()
+    unittest.main()