Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

DM-23426: Use PipelineTask test framework for MetricTask #67

Merged
merged 7 commits into from
Apr 2, 2020
Merged

DM-23426: Use PipelineTask test framework for MetricTask #67

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
0837d60
Use lsst.daf.butler.tests.
kfindeisen Mar 26, 2020
7baf184
Use lsst.pipe.base.testUtils.
kfindeisen Mar 26, 2020
efae434
Remove tests/butler_utils.py.
kfindeisen Mar 27, 2020
4ce4319
Validate MetricTask.run output for specific tasks.
kfindeisen Mar 30, 2020
681379b
Factor runQuantum test to handle multiple test cases.
kfindeisen Mar 30, 2020
4506b44
Test that ApdmMetricTask.runQuantum handles errors.
kfindeisen Mar 30, 2020
8dbd5c8
Suppress deprecation warnings from tests of deprecated code.
kfindeisen Mar 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
132 changes: 0 additions & 132 deletions tests/butler_utils.py
View file
Open in desktop

This file was deleted.

152 changes: 94 additions & 58 deletions tests/test_apdbMetricTask.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,12 @@

import lsst.utils.tests
from lsst.pex.config import Config
from lsst.daf.butler import Quantum
from lsst.pipe.base import Task, Struct
import lsst.daf.butler.tests as butlerTests
from lsst.pipe.base import Task, Struct, testUtils

from lsst.verify import Measurement
from lsst.verify.tasks import ApdbMetricTask
from lsst.verify.tasks import ApdbMetricTask, MetricComputationError
from lsst.verify.tasks.testUtils import ApdbMetricTestCase
from butler_utils import make_test_butler, make_dataset_type, \
ref_from_connection, run_quantum


class DummyTask(ApdbMetricTask):
Expand Down Expand Up @@ -66,76 +64,114 @@ def run(self, _):
config.validate()
return DummyTask(config=config)

@classmethod
def setUpClass(cls):
super().setUpClass()

cls.CAMERA_ID = "NotACam"
cls.VISIT_ID = 42
cls.CHIP_ID = 5

# makeTestRepo called in setUpClass because it's *very* slow
cls.root = tempfile.mkdtemp()
cls.repo = butlerTests.makeTestRepo(cls.root, {
"instrument": [cls.CAMERA_ID],
"visit": [cls.VISIT_ID],
"detector": [cls.CHIP_ID],
})

# self.task not visible at class level
task = cls.makeTask()
connections = task.config.ConnectionsClass(config=task.config)

butlerTests.addDatasetType(
cls.repo,
connections.measurement.name,
connections.measurement.dimensions,
connections.measurement.storageClass)
butlerTests.addDatasetType(
cls.repo,
connections.dbInfo.name,
connections.dbInfo.dimensions,
connections.dbInfo.storageClass)

@classmethod
def tearDownClass(cls):
shutil.rmtree(cls.root, ignore_errors=True)
super().tearDownClass()

def setUp(self):
super().setUp()

self.connections = self.task.config.ConnectionsClass(
config=self.task.config)
self.CAMERA_ID = "NotACam"
self.VISIT_ID = 42
self.CHIP_ID = 5

def _makeButler(self):
"""Construct a repository that supports the inputs and outputs of a
generic `ApdbMetricTask`.

This method is *very* slow; call it only from tests that need it.
"""

root = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, root, ignore_errors=True)
butler = make_test_butler(
root,
{
"instrument": [{"name": self.CAMERA_ID}],
"visit": [{"id": self.VISIT_ID,
"name": "only_visit",
"instrument": self.CAMERA_ID}],
"detector": [{"id": self.CHIP_ID,
"full_name": "only_ccd",
"instrument": self.CAMERA_ID}],
})
make_dataset_type(
butler,
self.connections.measurement.name,
self.connections.measurement.dimensions,
self.connections.measurement.storageClass)
make_dataset_type(
butler,
self.connections.dbInfo.name,
self.connections.dbInfo.dimensions,
self.connections.dbInfo.storageClass)
return butler

def testRunQuantum(self):
def _prepareQuantum(self, task):
inputId = {
"instrument": self.CAMERA_ID,
"visit": self.VISIT_ID,
"detector": self.CHIP_ID,
}

butler = self._makeButler()
# self.task.config not persistable because it refers to a local class
butler = butlerTests.makeTestCollection(self.repo)
# task.config not persistable if it refers to a local class
# We don't actually use the persisted config, so just make a new one
butler.put(self.task.ConfigClass(), "apdb_marker", inputId)
info = task.ConfigClass()
butler.put(info, "apdb_marker", inputId)

quantum = testUtils.makeQuantum(
task, butler, inputId,
{"dbInfo": [inputId], "measurement": inputId})

quantum = Quantum(taskClass=self.taskClass)
quantum.addPredictedInput(ref_from_connection(
butler,
self.connections.dbInfo,
inputId))
quantum.addOutput(ref_from_connection(
butler,
self.connections.measurement,
{"instrument": self.CAMERA_ID, }))
return (butler, quantum, info)

def testRunQuantum(self):
butler, quantum, input = self._prepareQuantum(self.task)

run_quantum(self.task, butler, quantum)
run = testUtils.runTestQuantum(self.task, butler, quantum)

# Did output data ID get passed to DummyTask.run?
measurement = butler.get(self.connections.measurement.name,
instrument=self.CAMERA_ID)
self.assertEqual(measurement.quantity,
len(self.CAMERA_ID) * u.dimensionless_unscaled)
expectedId = lsst.daf.butler.DataCoordinate.standardize(
{"instrument": self.CAMERA_ID},
universe=butler.registry.dimensions)
run.assert_called_once_with(
dbInfo=[input],
outputDataId=expectedId)

def testRunQuantumNone(self):
class NoneTask(DummyTask):
def run(self, *args, **kwargs):
return Struct(measurement=None)

config = NoneTask.ConfigClass()
config.connections.package = "verify"
config.connections.metric = "DummyApdb"
task = NoneTask(config=config)
butler, quantum, input = self._prepareQuantum(task)

with unittest.mock.patch.object(
lsst.pipe.base.ButlerQuantumContext, "put") as put:
testUtils.runTestQuantum(task, butler, quantum, mockRun=False)
# Should not attempt to write nonexistent data
put.assert_not_called()

def testRunQuantumException(self):
class ExceptionalTask(DummyTask):
def run(self, *args, **kwargs):
raise MetricComputationError()

config = ExceptionalTask.ConfigClass()
config.connections.package = "verify"
config.connections.metric = "DummyApdb"
task = ExceptionalTask(config=config)
butler, quantum, input = self._prepareQuantum(task)

with unittest.mock.patch.object(
lsst.pipe.base.ButlerQuantumContext, "put") as put:
testUtils.runTestQuantum(task, butler, quantum, mockRun=False)
# Should not propagate MetricComputationError
# Should not attempt to write data that was never returned
put.assert_not_called()


# Hack around unittest's hacky test setup system
Expand Down
12 changes: 10 additions & 2 deletions tests/test_commonMetrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ def setUp(self):

def testValid(self):
result = self.task.run(self.scienceTask.getFullMetadata())
lsst.pipe.base.testUtils.assertValidOutput(self.task, result)
meas = result.measurement

self.assertIsInstance(meas, Measurement)
Expand All @@ -86,13 +87,15 @@ def testNoMetric(self):

def testMissingData(self):
result = self.task.run(None)
lsst.pipe.base.testUtils.assertValidOutput(self.task, result)
meas = result.measurement
self.assertIsNone(meas)

def testRunDifferentMethod(self):
self.config.target = DummyTask._DefaultName + ".runDataRef"
task = TimingMetricTask(config=self.config)
result = task.run(self.scienceTask.getFullMetadata())
lsst.pipe.base.testUtils.assertValidOutput(task, result)
meas = result.measurement
self.assertIsNone(meas)

Expand Down Expand Up @@ -121,7 +124,8 @@ def testBadlyTypedKeys(self):
task.run(metadata)

def testDeprecated(self):
self.config.metric = "verify.DummyTime"
with warnings.catch_warnings(record=True):
self.config.metric = "verify.DummyTime"
self.config.connections.package = ""
self.config.connections.metric = ""
with warnings.catch_warnings(record=True) as emitted:
Expand Down Expand Up @@ -156,6 +160,7 @@ def setUp(self):

def testValid(self):
result = self.task.run(self.scienceTask.getFullMetadata())
lsst.pipe.base.testUtils.assertValidOutput(self.task, result)
meas = result.measurement

self.assertIsInstance(meas, Measurement)
Expand All @@ -171,13 +176,15 @@ def testNoMetric(self):

def testMissingData(self):
result = self.task.run(None)
lsst.pipe.base.testUtils.assertValidOutput(self.task, result)
meas = result.measurement
self.assertIsNone(meas)

def testRunDifferentMethod(self):
self.config.target = DummyTask._DefaultName + ".runDataRef"
task = MemoryMetricTask(config=self.config)
result = task.run(self.scienceTask.getFullMetadata())
lsst.pipe.base.testUtils.assertValidOutput(task, result)
meas = result.measurement
self.assertIsNone(meas)

Expand All @@ -194,7 +201,8 @@ def testBadlyTypedKeys(self):
task.run(metadata)

def testDeprecated(self):
self.config.metric = "verify.DummyMemory"
with warnings.catch_warnings(record=True):
self.config.metric = "verify.DummyMemory"
self.config.connections.package = ""
self.config.connections.metric = ""
with warnings.catch_warnings(record=True) as emitted:
Expand Down