Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DM-40798: Exclude sky sources from RB classification #210

Merged
merged 1 commit into from
Feb 7, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion pipelines/_ingredients/MetricsRuntime.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,13 @@ tasks:
connections.metric: PackageAlertsTime
connections.labelName: diaPipe
target: diaPipe:alertPackager.run
timing_filterDiaSrcCat:
class: lsst.verify.tasks.commonMetrics.TimingMetricTask
config:
connections.package: ap_association
connections.metric: FilterDiaSourceCatalogTime
connections.labelName: filterDiaSrcCat
target: filterDiaSrcCat.run
cputiming_isr:
class: lsst.verify.tasks.commonMetrics.CpuTimingMetricTask
config:
Expand Down Expand Up @@ -233,6 +240,13 @@ tasks:
connections.metric: PackageAlertsCpuTime
connections.labelName: diaPipe
target: diaPipe:alertPackager.run
cputiming_filterDiaSrcCat:
class: lsst.verify.tasks.commonMetrics.TimingMetricTask
config:
connections.package: ap_association
connections.metric: FilterDiaSourceCatalogCpuTime
connections.labelName: filterDiaSrcCat
target: filterDiaSrcCat.run
memory_apPipe:
class: lsst.verify.tasks.commonMetrics.MemoryMetricTask
config:
Expand All @@ -253,4 +267,4 @@ tasks:
connections.package: ap_association
connections.metric: PackageAlertsMemory
connections.labelName: diaPipe
target: diaPipe:alertPackager.run
target: diaPipe:alertPackager.run
27 changes: 26 additions & 1 deletion python/lsst/ap/verify/testPipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,14 @@
import lsst.afw.image as afwImage
import lsst.afw.math as afwMath
import lsst.afw.table as afwTable
from lsst.ap.association import (TransformDiaSourceCatalogConfig,
DiaPipelineConfig, FilterDiaSourceCatalogConfig)
from lsst.pipe.base import PipelineTask, Struct
from lsst.ip.isr import IsrTaskConfig
from lsst.ip.diffim import GetTemplateConfig, AlardLuptonSubtractConfig, DetectAndMeasureConfig
from lsst.pipe.tasks.characterizeImage import CharacterizeImageConfig
from lsst.pipe.tasks.calibrate import CalibrateConfig
from lsst.meas.transiNet import RBTransiNetConfig
from lsst.ap.association import TransformDiaSourceCatalogConfig, DiaPipelineConfig


class MockIsrTask(PipelineTask):
Expand Down Expand Up @@ -434,6 +435,30 @@ def run(self, science, matchedTemplate, difference,
)


class MockFilterDiaSourceCatalogTask(PipelineTask):
"""A do-nothing substitute for FilterDiaSourceCatalogTask.
"""
ConfigClass = FilterDiaSourceCatalogConfig
_DefaultName = "notFilterDiaSourceCatalog"

def run(self, diaSourceCat):
"""Produce filtering outputs with no processing.

Parameters
----------
diaSourceCat : `lsst.afw.table.SourceCatalog`
Catalog of sources measured on the difference image.

Returns
-------
results : `lsst.pipe.base.Struct`
Results struct with components.
"""
return Struct(filteredDiaSourceCat=afwTable.SourceCatalog(),
rejectedDiaSources=afwTable.SourceCatalog(),
)


class MockRBTransiNetTask(PipelineTask):
"""A do-nothing substitute for RBTransiNetTask.
"""
Expand Down
9 changes: 8 additions & 1 deletion tests/MockApPipe.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,11 @@ tasks:
config:
connections.coaddName: parameters.coaddName
doSkySources: True
filterDiaSrcCat:
class: lsst.ap.verify.testPipeline.MockFilterDiaSourceCatalogTask
config:
doRemoveSkySources: True
connections.coaddName: parameters.coaddName
rbClassify:
class: lsst.ap.verify.testPipeline.MockRBTransiNetTask
config:
Expand Down Expand Up @@ -65,11 +70,13 @@ contracts:
diaPipe.connections.ConnectionsClass(config=diaPipe).exposure.name
- detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
rbClassify.connections.ConnectionsClass(config=rbClassify).difference.name
- detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).diaSources.name ==
- filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).filteredDiaSourceCat.name ==
rbClassify.connections.ConnectionsClass(config=rbClassify).diaSources.name
- detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diffIm.name
- detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).diaSources.name ==
filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).diaSourceCat.name
- filterDiaSrcCat.connections.ConnectionsClass(config=filterDiaSrcCat).filteredDiaSourceCat.name ==
transformDiaSrcCat.connections.ConnectionsClass(config=transformDiaSrcCat).diaSourceCat.name
- detectAndMeasure.connections.ConnectionsClass(config=detectAndMeasure).subtractedMeasuredExposure.name ==
diaPipe.connections.ConnectionsClass(config=diaPipe).diffIm.name
Expand Down
14 changes: 11 additions & 3 deletions tests/test_testPipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
from lsst.ap.verify.testPipeline import MockIsrTask, MockCharacterizeImageTask, \
MockCalibrateTask, MockGetTemplateTask, \
MockAlardLuptonSubtractTask, MockDetectAndMeasureTask, MockTransformDiaSourceCatalogTask, \
MockRBTransiNetTask, MockDiaPipelineTask
MockRBTransiNetTask, MockDiaPipelineTask, MockFilterDiaSourceCatalogTask


class MockTaskTestSuite(unittest.TestCase):
Expand Down Expand Up @@ -121,6 +121,8 @@ def setUpClass(cls):
butlerTests.addDatasetType(cls.repo, "deepDiff_matchedExp", cls.visitId.dimensions, "ExposureF")
butlerTests.addDatasetType(cls.repo, "deepDiff_diaSrc", cls.visitId.dimensions, "SourceCatalog")
butlerTests.addDatasetType(cls.repo, "deepDiff_diaSrcTable", cls.visitId.dimensions, "DataFrame")
butlerTests.addDatasetType(cls.repo, "deepDiff_candidateDiaSrc", cls.visitId.dimensions,
"SourceCatalog")
butlerTests.addDatasetType(cls.repo, "visitSsObjects", cls.visitOnlyId.dimensions, "DataFrame")
butlerTests.addDatasetType(cls.repo, "apdb_marker", cls.visitId.dimensions, "Config")
butlerTests.addDatasetType(cls.repo, "deepDiff_assocDiaSrc", cls.visitId.dimensions, "DataFrame")
Expand Down Expand Up @@ -251,6 +253,12 @@ def testMockDetectAndMeasureTask(self):
})
pipelineTests.runTestQuantum(task, self.butler, quantum, mockRun=False)

def testMockFilterDiaSourceCatalogTask(self):
task = MockFilterDiaSourceCatalogTask()
pipelineTests.assertValidInitOutput(task)
result = task.run(afwTable.SourceCatalog())
pipelineTests.assertValidOutput(task, result)

def testMockRBTransiNetTask(self):
task = MockRBTransiNetTask()
pipelineTests.assertValidInitOutput(task)
Expand All @@ -264,7 +272,7 @@ def testMockRBTransiNetTask(self):
self.butler.put(afwImage.ExposureF(), "calexp", self.visitId)
self.butler.put(afwImage.ExposureF(), "deepDiff_differenceExp", self.visitId)
self.butler.put(afwImage.ExposureF(), "deepDiff_templateExp", self.visitId)
self.butler.put(afwTable.SourceCatalog(), "deepDiff_diaSrc", self.visitId)
self.butler.put(afwTable.SourceCatalog(), "deepDiff_candidateDiaSrc", self.visitId)
quantum = pipelineTests.makeQuantum(
task, self.butler, self.visitId,
{"template": self.visitId,
Expand All @@ -282,7 +290,7 @@ def testMockTransformDiaSourceCatalogTask(self):
result = task.run(afwTable.SourceCatalog(), afwImage.ExposureF(), 'k', 42)
pipelineTests.assertValidOutput(task, result)

self.butler.put(afwTable.SourceCatalog(), "deepDiff_diaSrc", self.visitId)
self.butler.put(afwTable.SourceCatalog(), "deepDiff_candidateDiaSrc", self.visitId)
self.butler.put(afwImage.ExposureF(), "deepDiff_differenceExp", self.visitId)
quantum = pipelineTests.makeQuantum(
task, self.butler, self.visitId,
Expand Down
Loading