Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

DM-20974: Remove aggregation support/requirement from MetricTask #53

Merged
merged 2 commits into from
Sep 5, 2019
Merged

DM-20974: Remove aggregation support/requirement from MetricTask #53

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4c3e31c
Pass only scalars to PpdbMetricTask.
kfindeisen Aug 30, 2019
21180e0
Convert metadata-based metrics to scalar input.
kfindeisen Aug 30, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
95 changes: 40 additions & 55 deletions python/lsst/ap/association/metrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,9 @@ def makeMeasurement(self, values):

Parameters
----------
values : sequence [`dict` [`str`, `int` or `None`]]
A list where each element corresponds to a metadata object passed
to `run`. Each `dict` has the following key:
values : `dict` [`str`, `int` or `None`]
A `dict` representation of the metadata. Each `dict` has the
following key:

``"newObjects"``
The number of new objects created for this image (`int`
Expand All @@ -60,19 +60,14 @@ def makeMeasurement(self, values):
measurement : `lsst.verify.Measurement` or `None`
The total number of new objects.
"""
nNew = 0
associated = False
for value in values:
if value["newObjects"] is not None:
try:
nNew += value["newObjects"]
except TypeError as e:
raise MetricComputationError("Corrupted value of numNewDiaObjects") from e
associated = True

if associated:
return Measurement(self.getOutputMetricName(self.config),
nNew * u.count)
if values["newObjects"] is not None:
try:
nNew = int(values["newObjects"])
except (ValueError, TypeError) as e:
raise MetricComputationError("Corrupted value of numNewDiaObjects") from e
else:
return Measurement(self.getOutputMetricName(self.config),
nNew * u.count)
else:
self.log.info("Nothing to do: no association results found.")
return None
Expand All @@ -98,9 +93,9 @@ def makeMeasurement(self, values):

Parameters
----------
values : sequence [`dict` [`str`, `int` or `None`]]
A list where each element corresponds to a metadata object passed
to `run`. Each `dict` has the following key:
values : `dict` [`str`, `int` or `None`]
A `dict` representation of the metadata. Each `dict` has the
following key:

``"unassociatedObjects"``
The number of DIAObjects not associated with a DiaSource in
Expand All @@ -112,19 +107,14 @@ def makeMeasurement(self, values):
measurement : `lsst.verify.Measurement` or `None`
The total number of unassociated objects.
"""
nNew = 0
associated = False
for value in values:
if value["unassociatedObjects"] is not None:
try:
nNew += value["unassociatedObjects"]
except TypeError as e:
raise MetricComputationError("Corrupted value of numUnassociatedDiaObjects") from e
associated = True

if associated:
return Measurement(self.getOutputMetricName(self.config),
nNew * u.count)
if values["unassociatedObjects"] is not None:
try:
nNew = int(values["unassociatedObjects"])
except (ValueError, TypeError) as e:
raise MetricComputationError("Corrupted value of numUnassociatedDiaObjects") from e
else:
return Measurement(self.getOutputMetricName(self.config),
nNew * u.count)
else:
self.log.info("Nothing to do: no association results found.")
return None
Expand All @@ -150,9 +140,9 @@ def makeMeasurement(self, values):

Parameters
----------
values : sequence [`dict` [`str`, `int` or `None`]]
A list where each element corresponds to a metadata object passed
to `run`. Each `dict` has the following keys:
values : `dict` [`str`, `int` or `None`]
A `dict` representation of the metadata. Each `dict` has the
following keys:

``"updatedObjects"``
The number of DIAObjects updated for this image (`int` or
Expand All @@ -168,27 +158,22 @@ def makeMeasurement(self, values):
measurement : `lsst.verify.Measurement` or `None`
The total number of unassociated objects.
"""
nUpdated = 0
nUnassociated = 0
associated = False
for value in values:
if value["updatedObjects"] is not None \
and value["unassociatedObjects"] is not None:
try:
nUpdated += value["updatedObjects"]
nUnassociated += value["unassociatedObjects"]
except TypeError as e:
raise MetricComputationError("Corrupted value of numUpdatedDiaObjects "
"or numUnassociatedDiaObjects") from e
associated = True

if associated:
if nUpdated <= 0 and nUnassociated <= 0:
raise MetricComputationError("No pre-existing DIAObjects; can't compute updated fraction.")
if values["updatedObjects"] is not None \
and values["unassociatedObjects"] is not None:
try:
nUpdated = int(values["updatedObjects"])
nUnassociated = int(values["unassociatedObjects"])
except (ValueError, TypeError) as e:
raise MetricComputationError("Corrupted value of numUpdatedDiaObjects "
"or numUnassociatedDiaObjects") from e
else:
fraction = nUpdated / (nUpdated + nUnassociated)
return Measurement(self.getOutputMetricName(self.config),
fraction * u.dimensionless_unscaled)
if nUpdated <= 0 and nUnassociated <= 0:
raise MetricComputationError(
"No pre-existing DIAObjects; can't compute updated fraction.")
else:
fraction = nUpdated / (nUpdated + nUnassociated)
return Measurement(self.getOutputMetricName(self.config),
fraction * u.dimensionless_unscaled)
else:
self.log.info("Nothing to do: no association results found.")
return None
Expand Down
104 changes: 28 additions & 76 deletions tests/test_metrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,32 +58,27 @@ def makeTask(cls):

def testValid(self):
metadata = _makeAssociationMetadata()
result = self.task.run([metadata])
result = self.task.run(metadata)
meas = result.measurement

self.assertEqual(meas.metric_name, Name(metric="ap_association.numNewDiaObjects"))
self.assertEqual(meas.quantity, metadata.getAsDouble("association.numNewDiaObjects") * u.count)

def testNoNew(self):
metadata = _makeAssociationMetadata(numNew=0)
result = self.task.run([metadata])
result = self.task.run(metadata)
meas = result.measurement

self.assertEqual(meas.metric_name, Name(metric="ap_association.numNewDiaObjects"))
self.assertEqual(meas.quantity, 0.0 * u.count)

def testMissingData(self):
result = self.task.run([None])
meas = result.measurement
self.assertIsNone(meas)

def testNoDataExpected(self):
result = self.task.run([])
result = self.task.run(None)
meas = result.measurement
self.assertIsNone(meas)

def testAssociationFailed(self):
result = self.task.run([PropertySet()])
result = self.task.run(PropertySet())
meas = result.measurement
self.assertIsNone(meas)

Expand All @@ -92,7 +87,7 @@ def testBadlyTypedKeys(self):
metadata.set("association.numNewDiaObjects", "Ultimate Answer")

with self.assertRaises(MetricComputationError):
self.task.run([metadata])
self.task.run(metadata)

def testGetInputDatasetTypes(self):
config = self.taskClass.ConfigClass()
Expand All @@ -104,25 +99,14 @@ def testGetInputDatasetTypes(self):

def testFineGrainedMetric(self):
metadata = _makeAssociationMetadata()
inputData = {"metadata": [metadata]}
inputDataIds = {"metadata": [{"visit": 42, "ccd": 1}]}
inputData = {"metadata": metadata}
inputDataIds = {"metadata": {"visit": 42, "ccd": 1}}
outputDataId = {"measurement": {"visit": 42, "ccd": 1}}
measDirect = self.task.run([metadata]).measurement
measDirect = self.task.run(metadata).measurement
measIndirect = self.task.adaptArgsAndRun(inputData, inputDataIds, outputDataId).measurement

assert_quantity_allclose(measIndirect.quantity, measDirect.quantity)

def testCoarseGrainedMetric(self):
metadata = _makeAssociationMetadata()
nCcds = 3
inputData = {"metadata": [metadata] * nCcds}
inputDataIds = {"metadata": [{"visit": 42, "ccd": x} for x in range(nCcds)]}
outputDataId = {"measurement": {"visit": 42}}
measDirect = self.task.run([metadata]).measurement
measMany = self.task.adaptArgsAndRun(inputData, inputDataIds, outputDataId).measurement

assert_quantity_allclose(measMany.quantity, nCcds * measDirect.quantity)


class TestUnassociatedDiaObjects(MetadataMetricTestCase):

Expand All @@ -132,7 +116,7 @@ def makeTask(cls):

def testValid(self):
metadata = _makeAssociationMetadata()
result = self.task.run([metadata])
result = self.task.run(metadata)
meas = result.measurement

self.assertEqual(meas.metric_name, Name(metric="ap_association.numUnassociatedDiaObjects"))
Expand All @@ -141,24 +125,19 @@ def testValid(self):

def testAllUpdated(self):
metadata = _makeAssociationMetadata(numUnassociated=0)
result = self.task.run([metadata])
result = self.task.run(metadata)
meas = result.measurement

self.assertEqual(meas.metric_name, Name(metric="ap_association.numUnassociatedDiaObjects"))
self.assertEqual(meas.quantity, 0.0 * u.count)

def testMissingData(self):
result = self.task.run([None])
meas = result.measurement
self.assertIsNone(meas)

def testNoDataExpected(self):
result = self.task.run([])
result = self.task.run(None)
meas = result.measurement
self.assertIsNone(meas)

def testAssociationFailed(self):
result = self.task.run([PropertySet()])
result = self.task.run(PropertySet())
meas = result.measurement
self.assertIsNone(meas)

Expand All @@ -167,7 +146,7 @@ def testBadlyTypedKeys(self):
metadata.set("association.numUnassociatedDiaObjects", "Ultimate Answer")

with self.assertRaises(MetricComputationError):
self.task.run([metadata])
self.task.run(metadata)

def testGetInputDatasetTypes(self):
config = self.taskClass.ConfigClass()
Expand All @@ -179,25 +158,14 @@ def testGetInputDatasetTypes(self):

def testFineGrainedMetric(self):
metadata = _makeAssociationMetadata()
inputData = {"metadata": [metadata]}
inputDataIds = {"metadata": [{"visit": 42, "ccd": 1}]}
inputData = {"metadata": metadata}
inputDataIds = {"metadata": {"visit": 42, "ccd": 1}}
outputDataId = {"measurement": {"visit": 42, "ccd": 1}}
measDirect = self.task.run([metadata]).measurement
measDirect = self.task.run(metadata).measurement
measIndirect = self.task.adaptArgsAndRun(inputData, inputDataIds, outputDataId).measurement

assert_quantity_allclose(measIndirect.quantity, measDirect.quantity)

def testCoarseGrainedMetric(self):
metadata = _makeAssociationMetadata()
nCcds = 3
inputData = {"metadata": [metadata] * nCcds}
inputDataIds = {"metadata": [{"visit": 42, "ccd": x} for x in range(nCcds)]}
outputDataId = {"measurement": {"visit": 42}}
measDirect = self.task.run([metadata]).measurement
measMany = self.task.adaptArgsAndRun(inputData, inputDataIds, outputDataId).measurement

assert_quantity_allclose(measMany.quantity, nCcds * measDirect.quantity)


class TestFracUpdatedDiaObjects(MetadataMetricTestCase):

Expand All @@ -207,7 +175,7 @@ def makeTask(cls):

def testValid(self):
metadata = _makeAssociationMetadata()
result = self.task.run([metadata])
result = self.task.run(metadata)
meas = result.measurement

self.assertEqual(meas.metric_name, Name(metric="ap_association.fracUpdatedDiaObjects"))
Expand All @@ -217,15 +185,15 @@ def testValid(self):

def testNoUpdated(self):
metadata = _makeAssociationMetadata(numUpdated=0)
result = self.task.run([metadata])
result = self.task.run(metadata)
meas = result.measurement

self.assertEqual(meas.metric_name, Name(metric="ap_association.fracUpdatedDiaObjects"))
self.assertEqual(meas.quantity, 0.0 * u.dimensionless_unscaled)

def testAllUpdated(self):
metadata = _makeAssociationMetadata(numUnassociated=0)
result = self.task.run([metadata])
result = self.task.run(metadata)
meas = result.measurement

self.assertEqual(meas.metric_name, Name(metric="ap_association.fracUpdatedDiaObjects"))
Expand All @@ -234,20 +202,15 @@ def testAllUpdated(self):
def testNoObjects(self):
metadata = _makeAssociationMetadata(numUpdated=0, numUnassociated=0)
with self.assertRaises(MetricComputationError):
self.task.run([metadata])
self.task.run(metadata)

def testMissingData(self):
result = self.task.run([None])
meas = result.measurement
self.assertIsNone(meas)

def testNoDataExpected(self):
result = self.task.run([])
result = self.task.run(None)
meas = result.measurement
self.assertIsNone(meas)

def testAssociationFailed(self):
result = self.task.run([PropertySet()])
result = self.task.run(PropertySet())
meas = result.measurement
self.assertIsNone(meas)

Expand All @@ -256,7 +219,7 @@ def testBadlyTypedKeys(self):
metadata.set("association.numUnassociatedDiaObjects", "Ultimate Answer")

with self.assertRaises(MetricComputationError):
self.task.run([metadata])
self.task.run(metadata)

def testGetInputDatasetTypes(self):
config = self.taskClass.ConfigClass()
Expand All @@ -268,25 +231,14 @@ def testGetInputDatasetTypes(self):

def testFineGrainedMetric(self):
metadata = _makeAssociationMetadata()
inputData = {"metadata": [metadata]}
inputDataIds = {"metadata": [{"visit": 42, "ccd": 1}]}
inputData = {"metadata": metadata}
inputDataIds = {"metadata": {"visit": 42, "ccd": 1}}
outputDataId = {"measurement": {"visit": 42, "ccd": 1}}
measDirect = self.task.run([metadata]).measurement
measDirect = self.task.run(metadata).measurement
measIndirect = self.task.adaptArgsAndRun(inputData, inputDataIds, outputDataId).measurement

assert_quantity_allclose(measIndirect.quantity, measDirect.quantity)

def testCoarseGrainedMetric(self):
metadata = _makeAssociationMetadata()
nCcds = 3
inputData = {"metadata": [metadata] * nCcds}
inputDataIds = {"metadata": [{"visit": 42, "ccd": x} for x in range(nCcds)]}
outputDataId = {"measurement": {"visit": 42}}
measDirect = self.task.run([metadata]).measurement
measMany = self.task.adaptArgsAndRun(inputData, inputDataIds, outputDataId).measurement

assert_quantity_allclose(measMany.quantity, measDirect.quantity)


class TestTotalUnassociatedObjects(PpdbMetricTestCase):

Expand Down Expand Up @@ -329,7 +281,7 @@ def setUp(self):
# Do the patch here to avoid passing extra arguments to superclass tests

def testValid(self):
result = self.task.adaptArgsAndRun({"dbInfo": [{"test_value": 42}]},
result = self.task.adaptArgsAndRun({"dbInfo": {"test_value": 42}},
{"dbInfo": {}},
{"measurement": {}})
meas = result.measurement
Expand All @@ -338,7 +290,7 @@ def testValid(self):
self.assertEqual(meas.quantity, 42 * u.count)

def testAllAssociated(self):
result = self.task.adaptArgsAndRun({"dbInfo": [{"test_value": 0}]},
result = self.task.adaptArgsAndRun({"dbInfo": {"test_value": 0}},
{"dbInfo": {}},
{"measurement": {}})
meas = result.measurement
Expand Down