Skip to content

Commit

Permalink
Convert metadata-based metrics to scalar input.
Browse files Browse the repository at this point in the history
  • Loading branch information
@kfindeisen
kfindeisen committed Aug 30, 2019
1 parent 4c3e31c commit 21180e0
Show file tree
Hide file tree
Showing 2 changed files with 66 additions and 129 deletions.
95 changes: 40 additions & 55 deletions python/lsst/ap/association/metrics.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,9 @@ def makeMeasurement(self, values):
Parameters
----------
values : sequence [`dict` [`str`, `int` or `None`]]
A list where each element corresponds to a metadata object passed
to `run`. Each `dict` has the following key:
values : `dict` [`str`, `int` or `None`]
A `dict` representation of the metadata. Each `dict` has the
following key:
``"newObjects"``
The number of new objects created for this image (`int`
Expand All @@ -60,19 +60,14 @@ def makeMeasurement(self, values):
measurement : `lsst.verify.Measurement` or `None`
The total number of new objects.
"""
nNew = 0
associated = False
for value in values:
if value["newObjects"] is not None:
try:
nNew += value["newObjects"]
except TypeError as e:
raise MetricComputationError("Corrupted value of numNewDiaObjects") from e
associated = True

if associated:
return Measurement(self.getOutputMetricName(self.config),
nNew * u.count)
if values["newObjects"] is not None:
try:
nNew = int(values["newObjects"])
except (ValueError, TypeError) as e:
raise MetricComputationError("Corrupted value of numNewDiaObjects") from e
else:
return Measurement(self.getOutputMetricName(self.config),
nNew * u.count)
else:
self.log.info("Nothing to do: no association results found.")
return None
Expand All @@ -98,9 +93,9 @@ def makeMeasurement(self, values):
Parameters
----------
values : sequence [`dict` [`str`, `int` or `None`]]
A list where each element corresponds to a metadata object passed
to `run`. Each `dict` has the following key:
values : `dict` [`str`, `int` or `None`]
A `dict` representation of the metadata. Each `dict` has the
following key:
``"unassociatedObjects"``
The number of DIAObjects not associated with a DiaSource in
Expand All @@ -112,19 +107,14 @@ def makeMeasurement(self, values):
measurement : `lsst.verify.Measurement` or `None`
The total number of unassociated objects.
"""
nNew = 0
associated = False
for value in values:
if value["unassociatedObjects"] is not None:
try:
nNew += value["unassociatedObjects"]
except TypeError as e:
raise MetricComputationError("Corrupted value of numUnassociatedDiaObjects") from e
associated = True

if associated:
return Measurement(self.getOutputMetricName(self.config),
nNew * u.count)
if values["unassociatedObjects"] is not None:
try:
nNew = int(values["unassociatedObjects"])
except (ValueError, TypeError) as e:
raise MetricComputationError("Corrupted value of numUnassociatedDiaObjects") from e
else:
return Measurement(self.getOutputMetricName(self.config),
nNew * u.count)
else:
self.log.info("Nothing to do: no association results found.")
return None
Expand All @@ -150,9 +140,9 @@ def makeMeasurement(self, values):
Parameters
----------
values : sequence [`dict` [`str`, `int` or `None`]]
A list where each element corresponds to a metadata object passed
to `run`. Each `dict` has the following keys:
values : `dict` [`str`, `int` or `None`]
A `dict` representation of the metadata. Each `dict` has the
following keys:
``"updatedObjects"``
The number of DIAObjects updated for this image (`int` or
Expand All @@ -168,27 +158,22 @@ def makeMeasurement(self, values):
measurement : `lsst.verify.Measurement` or `None`
The total number of unassociated objects.
"""
nUpdated = 0
nUnassociated = 0
associated = False
for value in values:
if value["updatedObjects"] is not None \
and value["unassociatedObjects"] is not None:
try:
nUpdated += value["updatedObjects"]
nUnassociated += value["unassociatedObjects"]
except TypeError as e:
raise MetricComputationError("Corrupted value of numUpdatedDiaObjects "
"or numUnassociatedDiaObjects") from e
associated = True

if associated:
if nUpdated <= 0 and nUnassociated <= 0:
raise MetricComputationError("No pre-existing DIAObjects; can't compute updated fraction.")
if values["updatedObjects"] is not None \
and values["unassociatedObjects"] is not None:
try:
nUpdated = int(values["updatedObjects"])
nUnassociated = int(values["unassociatedObjects"])
except (ValueError, TypeError) as e:
raise MetricComputationError("Corrupted value of numUpdatedDiaObjects "
"or numUnassociatedDiaObjects") from e
else:
fraction = nUpdated / (nUpdated + nUnassociated)
return Measurement(self.getOutputMetricName(self.config),
fraction * u.dimensionless_unscaled)
if nUpdated <= 0 and nUnassociated <= 0:
raise MetricComputationError(
"No pre-existing DIAObjects; can't compute updated fraction.")
else:
fraction = nUpdated / (nUpdated + nUnassociated)
return Measurement(self.getOutputMetricName(self.config),
fraction * u.dimensionless_unscaled)
else:
self.log.info("Nothing to do: no association results found.")
return None
Expand Down
100 changes: 26 additions & 74 deletions tests/test_metrics.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,32 +58,27 @@ def makeTask(cls):

def testValid(self):
metadata = _makeAssociationMetadata()
result = self.task.run([metadata])
result = self.task.run(metadata)
meas = result.measurement

self.assertEqual(meas.metric_name, Name(metric="ap_association.numNewDiaObjects"))
self.assertEqual(meas.quantity, metadata.getAsDouble("association.numNewDiaObjects") * u.count)

def testNoNew(self):
metadata = _makeAssociationMetadata(numNew=0)
result = self.task.run([metadata])
result = self.task.run(metadata)
meas = result.measurement

self.assertEqual(meas.metric_name, Name(metric="ap_association.numNewDiaObjects"))
self.assertEqual(meas.quantity, 0.0 * u.count)

def testMissingData(self):
result = self.task.run([None])
meas = result.measurement
self.assertIsNone(meas)

def testNoDataExpected(self):
result = self.task.run([])
result = self.task.run(None)
meas = result.measurement
self.assertIsNone(meas)

def testAssociationFailed(self):
result = self.task.run([PropertySet()])
result = self.task.run(PropertySet())
meas = result.measurement
self.assertIsNone(meas)

Expand All @@ -92,7 +87,7 @@ def testBadlyTypedKeys(self):
metadata.set("association.numNewDiaObjects", "Ultimate Answer")

with self.assertRaises(MetricComputationError):
self.task.run([metadata])
self.task.run(metadata)

def testGetInputDatasetTypes(self):
config = self.taskClass.ConfigClass()
Expand All @@ -104,25 +99,14 @@ def testGetInputDatasetTypes(self):

def testFineGrainedMetric(self):
metadata = _makeAssociationMetadata()
inputData = {"metadata": [metadata]}
inputDataIds = {"metadata": [{"visit": 42, "ccd": 1}]}
inputData = {"metadata": metadata}
inputDataIds = {"metadata": {"visit": 42, "ccd": 1}}
outputDataId = {"measurement": {"visit": 42, "ccd": 1}}
measDirect = self.task.run([metadata]).measurement
measDirect = self.task.run(metadata).measurement
measIndirect = self.task.adaptArgsAndRun(inputData, inputDataIds, outputDataId).measurement

assert_quantity_allclose(measIndirect.quantity, measDirect.quantity)

def testCoarseGrainedMetric(self):
metadata = _makeAssociationMetadata()
nCcds = 3
inputData = {"metadata": [metadata] * nCcds}
inputDataIds = {"metadata": [{"visit": 42, "ccd": x} for x in range(nCcds)]}
outputDataId = {"measurement": {"visit": 42}}
measDirect = self.task.run([metadata]).measurement
measMany = self.task.adaptArgsAndRun(inputData, inputDataIds, outputDataId).measurement

assert_quantity_allclose(measMany.quantity, nCcds * measDirect.quantity)


class TestUnassociatedDiaObjects(MetadataMetricTestCase):

Expand All @@ -132,7 +116,7 @@ def makeTask(cls):

def testValid(self):
metadata = _makeAssociationMetadata()
result = self.task.run([metadata])
result = self.task.run(metadata)
meas = result.measurement

self.assertEqual(meas.metric_name, Name(metric="ap_association.numUnassociatedDiaObjects"))
Expand All @@ -141,24 +125,19 @@ def testValid(self):

def testAllUpdated(self):
metadata = _makeAssociationMetadata(numUnassociated=0)
result = self.task.run([metadata])
result = self.task.run(metadata)
meas = result.measurement

self.assertEqual(meas.metric_name, Name(metric="ap_association.numUnassociatedDiaObjects"))
self.assertEqual(meas.quantity, 0.0 * u.count)

def testMissingData(self):
result = self.task.run([None])
meas = result.measurement
self.assertIsNone(meas)

def testNoDataExpected(self):
result = self.task.run([])
result = self.task.run(None)
meas = result.measurement
self.assertIsNone(meas)

def testAssociationFailed(self):
result = self.task.run([PropertySet()])
result = self.task.run(PropertySet())
meas = result.measurement
self.assertIsNone(meas)

Expand All @@ -167,7 +146,7 @@ def testBadlyTypedKeys(self):
metadata.set("association.numUnassociatedDiaObjects", "Ultimate Answer")

with self.assertRaises(MetricComputationError):
self.task.run([metadata])
self.task.run(metadata)

def testGetInputDatasetTypes(self):
config = self.taskClass.ConfigClass()
Expand All @@ -179,25 +158,14 @@ def testGetInputDatasetTypes(self):

def testFineGrainedMetric(self):
metadata = _makeAssociationMetadata()
inputData = {"metadata": [metadata]}
inputDataIds = {"metadata": [{"visit": 42, "ccd": 1}]}
inputData = {"metadata": metadata}
inputDataIds = {"metadata": {"visit": 42, "ccd": 1}}
outputDataId = {"measurement": {"visit": 42, "ccd": 1}}
measDirect = self.task.run([metadata]).measurement
measDirect = self.task.run(metadata).measurement
measIndirect = self.task.adaptArgsAndRun(inputData, inputDataIds, outputDataId).measurement

assert_quantity_allclose(measIndirect.quantity, measDirect.quantity)

def testCoarseGrainedMetric(self):
metadata = _makeAssociationMetadata()
nCcds = 3
inputData = {"metadata": [metadata] * nCcds}
inputDataIds = {"metadata": [{"visit": 42, "ccd": x} for x in range(nCcds)]}
outputDataId = {"measurement": {"visit": 42}}
measDirect = self.task.run([metadata]).measurement
measMany = self.task.adaptArgsAndRun(inputData, inputDataIds, outputDataId).measurement

assert_quantity_allclose(measMany.quantity, nCcds * measDirect.quantity)


class TestFracUpdatedDiaObjects(MetadataMetricTestCase):

Expand All @@ -207,7 +175,7 @@ def makeTask(cls):

def testValid(self):
metadata = _makeAssociationMetadata()
result = self.task.run([metadata])
result = self.task.run(metadata)
meas = result.measurement

self.assertEqual(meas.metric_name, Name(metric="ap_association.fracUpdatedDiaObjects"))
Expand All @@ -217,15 +185,15 @@ def testValid(self):

def testNoUpdated(self):
metadata = _makeAssociationMetadata(numUpdated=0)
result = self.task.run([metadata])
result = self.task.run(metadata)
meas = result.measurement

self.assertEqual(meas.metric_name, Name(metric="ap_association.fracUpdatedDiaObjects"))
self.assertEqual(meas.quantity, 0.0 * u.dimensionless_unscaled)

def testAllUpdated(self):
metadata = _makeAssociationMetadata(numUnassociated=0)
result = self.task.run([metadata])
result = self.task.run(metadata)
meas = result.measurement

self.assertEqual(meas.metric_name, Name(metric="ap_association.fracUpdatedDiaObjects"))
Expand All @@ -234,20 +202,15 @@ def testAllUpdated(self):
def testNoObjects(self):
metadata = _makeAssociationMetadata(numUpdated=0, numUnassociated=0)
with self.assertRaises(MetricComputationError):
self.task.run([metadata])
self.task.run(metadata)

def testMissingData(self):
result = self.task.run([None])
meas = result.measurement
self.assertIsNone(meas)

def testNoDataExpected(self):
result = self.task.run([])
result = self.task.run(None)
meas = result.measurement
self.assertIsNone(meas)

def testAssociationFailed(self):
result = self.task.run([PropertySet()])
result = self.task.run(PropertySet())
meas = result.measurement
self.assertIsNone(meas)

Expand All @@ -256,7 +219,7 @@ def testBadlyTypedKeys(self):
metadata.set("association.numUnassociatedDiaObjects", "Ultimate Answer")

with self.assertRaises(MetricComputationError):
self.task.run([metadata])
self.task.run(metadata)

def testGetInputDatasetTypes(self):
config = self.taskClass.ConfigClass()
Expand All @@ -268,25 +231,14 @@ def testGetInputDatasetTypes(self):

def testFineGrainedMetric(self):
metadata = _makeAssociationMetadata()
inputData = {"metadata": [metadata]}
inputDataIds = {"metadata": [{"visit": 42, "ccd": 1}]}
inputData = {"metadata": metadata}
inputDataIds = {"metadata": {"visit": 42, "ccd": 1}}
outputDataId = {"measurement": {"visit": 42, "ccd": 1}}
measDirect = self.task.run([metadata]).measurement
measDirect = self.task.run(metadata).measurement
measIndirect = self.task.adaptArgsAndRun(inputData, inputDataIds, outputDataId).measurement

assert_quantity_allclose(measIndirect.quantity, measDirect.quantity)

def testCoarseGrainedMetric(self):
metadata = _makeAssociationMetadata()
nCcds = 3
inputData = {"metadata": [metadata] * nCcds}
inputDataIds = {"metadata": [{"visit": 42, "ccd": x} for x in range(nCcds)]}
outputDataId = {"measurement": {"visit": 42}}
measDirect = self.task.run([metadata]).measurement
measMany = self.task.adaptArgsAndRun(inputData, inputDataIds, outputDataId).measurement

assert_quantity_allclose(measMany.quantity, measDirect.quantity)


class TestTotalUnassociatedObjects(PpdbMetricTestCase):

Expand Down

0 comments on commit 21180e0

Please sign in to comment.