Merge pull request #210 from lsst/tickets/DM-40798

DM-40798: Exclude sky sources from RB classification

pipelines/_ingredients/MetricsRuntime.yaml

@@ -119,6 +119,13 @@ tasks:
       connections.metric: PackageAlertsTime
       connections.labelName: diaPipe
       target: diaPipe:alertPackager.run
+  timing_filterDiaSrcCat:
+    class: lsst.verify.tasks.commonMetrics.TimingMetricTask
+    config:
+      connections.package: ap_association
+      connections.metric: FilterDiaSourceCatalogTime
+      connections.labelName: filterDiaSrcCat
+      target: filterDiaSrcCat.run
   cputiming_isr:
     class: lsst.verify.tasks.commonMetrics.CpuTimingMetricTask
     config:
@@ -233,6 +240,13 @@ tasks:
       connections.metric: PackageAlertsCpuTime
       connections.labelName: diaPipe
       target: diaPipe:alertPackager.run
+  cputiming_filterDiaSrcCat:
+    class: lsst.verify.tasks.commonMetrics.TimingMetricTask
+    config:
+      connections.package: ap_association
+      connections.metric: FilterDiaSourceCatalogCpuTime
+      connections.labelName: filterDiaSrcCat
+      target: filterDiaSrcCat.run
   memory_apPipe:
     class: lsst.verify.tasks.commonMetrics.MemoryMetricTask
     config:
@@ -253,4 +267,4 @@ tasks:
       connections.package: ap_association
       connections.metric: PackageAlertsMemory
       connections.labelName: diaPipe
-      target: diaPipe:alertPackager.run
+      target: diaPipe:alertPackager.run

python/lsst/ap/verify/testPipeline.py

@@ -34,13 +34,14 @@
 import lsst.afw.image as afwImage
 import lsst.afw.math as afwMath
 import lsst.afw.table as afwTable
+from lsst.ap.association import (TransformDiaSourceCatalogConfig,
+                                 DiaPipelineConfig, FilterDiaSourceCatalogConfig)
 from lsst.pipe.base import PipelineTask, Struct
 from lsst.ip.isr import IsrTaskConfig
 from lsst.ip.diffim import GetTemplateConfig, AlardLuptonSubtractConfig, DetectAndMeasureConfig
 from lsst.pipe.tasks.characterizeImage import CharacterizeImageConfig
 from lsst.pipe.tasks.calibrate import CalibrateConfig
 from lsst.meas.transiNet import RBTransiNetConfig
-from lsst.ap.association import TransformDiaSourceCatalogConfig, DiaPipelineConfig
 
 
 class MockIsrTask(PipelineTask):
@@ -434,6 +435,30 @@ def run(self, science, matchedTemplate, difference,
                       )
 
 
+class MockFilterDiaSourceCatalogTask(PipelineTask):
+    """A do-nothing substitute for FilterDiaSourceCatalogTask.
+    """
+    ConfigClass = FilterDiaSourceCatalogConfig
+    _DefaultName = "notFilterDiaSourceCatalog"
+
+    def run(self, diaSourceCat):
+        """Produce filtering outputs with no processing.
+
+        Parameters
+        ----------
+        diaSourceCat : `lsst.afw.table.SourceCatalog`
+            Catalog of sources measured on the difference image.
+
+        Returns
+        -------
+        results : `lsst.pipe.base.Struct`
+            Results struct with components.
+        """
+        return Struct(filteredDiaSourceCat=afwTable.SourceCatalog(),
+                      rejectedDiaSources=afwTable.SourceCatalog(),
+                      )
+
+
 class MockRBTransiNetTask(PipelineTask):
     """A do-nothing substitute for RBTransiNetTask.
     """

tests/MockApPipe.yaml

@@ -35,6 +35,11 @@ tasks:
     config:
       connections.coaddName: parameters.coaddName
       doSkySources: True
+  filterDiaSrcCat:
+    class: lsst.ap.verify.testPipeline.MockFilterDiaSourceCatalogTask
+    config:
+      doRemoveSkySources: True
+      connections.coaddName: parameters.coaddName
   rbClassify:
     class: lsst.ap.verify.testPipeline.MockRBTransiNetTask
     config:
@@ -65,11 +70,13 @@ contracts:
       diaPipe.connections.ConnectionsClass(config=diaPipe).exposure.name
   - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
       rbClassify.connections.ConnectionsClass(config=rbClassify).difference.name
-  - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).diaSources.name ==
+  - filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).filteredDiaSourceCat.name ==
       rbClassify.connections.ConnectionsClass(config=rbClassify).diaSources.name
   - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
       transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diffIm.name
   - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).diaSources.name ==
+      filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).diaSourceCat.name
+  - filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).filteredDiaSourceCat.name ==
       transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diaSourceCat.name
   - detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
         diaPipe.connections.ConnectionsClass(config=diaPipe).diffIm.name

tests/test_testPipeline.py

@@ -38,7 +38,7 @@
 from lsst.ap.verify.testPipeline import MockIsrTask, MockCharacterizeImageTask, \
     MockCalibrateTask, MockGetTemplateTask, \
     MockAlardLuptonSubtractTask, MockDetectAndMeasureTask, MockTransformDiaSourceCatalogTask, \
-    MockRBTransiNetTask, MockDiaPipelineTask
+    MockRBTransiNetTask, MockDiaPipelineTask, MockFilterDiaSourceCatalogTask
 
 
 class MockTaskTestSuite(unittest.TestCase):
@@ -121,6 +121,8 @@ def setUpClass(cls):
         butlerTests.addDatasetType(cls.repo, "deepDiff_matchedExp", cls.visitId.dimensions, "ExposureF")
         butlerTests.addDatasetType(cls.repo, "deepDiff_diaSrc", cls.visitId.dimensions, "SourceCatalog")
         butlerTests.addDatasetType(cls.repo, "deepDiff_diaSrcTable", cls.visitId.dimensions, "DataFrame")
+        butlerTests.addDatasetType(cls.repo, "deepDiff_candidateDiaSrc", cls.visitId.dimensions,
+                                   "SourceCatalog")
         butlerTests.addDatasetType(cls.repo, "visitSsObjects", cls.visitOnlyId.dimensions, "DataFrame")
         butlerTests.addDatasetType(cls.repo, "apdb_marker", cls.visitId.dimensions, "Config")
         butlerTests.addDatasetType(cls.repo, "deepDiff_assocDiaSrc", cls.visitId.dimensions, "DataFrame")
@@ -251,6 +253,12 @@ def testMockDetectAndMeasureTask(self):
              })
         pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)
 
+    def testMockFilterDiaSourceCatalogTask(self):
+        task = MockFilterDiaSourceCatalogTask()
+        pipelineTests.assertValidInitOutput(task)
+        result = task.run(afwTable.SourceCatalog())
+        pipelineTests.assertValidOutput(task, result)
+
     def testMockRBTransiNetTask(self):
         task = MockRBTransiNetTask()
         pipelineTests.assertValidInitOutput(task)
@@ -264,7 +272,7 @@ def testMockRBTransiNetTask(self):
         self.butler.put(afwImage.ExposureF(), "calexp", self.visitId)
         self.butler.put(afwImage.ExposureF(), "deepDiff_differenceExp", self.visitId)
         self.butler.put(afwImage.ExposureF(), "deepDiff_templateExp", self.visitId)
-        self.butler.put(afwTable.SourceCatalog(), "deepDiff_diaSrc", self.visitId)
+        self.butler.put(afwTable.SourceCatalog(), "deepDiff_candidateDiaSrc", self.visitId)
         quantum = pipelineTests.makeQuantum(
             task, self.butler, self.visitId,
             {"template": self.visitId,
@@ -282,7 +290,7 @@ def testMockTransformDiaSourceCatalogTask(self):
         result = task.run(afwTable.SourceCatalog(), afwImage.ExposureF(), 'k', 42)
         pipelineTests.assertValidOutput(task, result)
 
-        self.butler.put(afwTable.SourceCatalog(), "deepDiff_diaSrc", self.visitId)
+        self.butler.put(afwTable.SourceCatalog(), "deepDiff_candidateDiaSrc", self.visitId)
         self.butler.put(afwImage.ExposureF(), "deepDiff_differenceExp", self.visitId)
         quantum = pipelineTests.makeQuantum(
             task, self.butler, self.visitId,