Skip to content

Commit

Permalink
Merge branch 'tickets/DM-28236' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
@kfindeisen
kfindeisen committed Jan 13, 2021
2 parents e80a327 + 12c8080 commit 6948e2c
Show file tree
Hide file tree
Showing 3 changed files with 139 additions and 73 deletions.
117 changes: 79 additions & 38 deletions python/lsst/obs/base/cameraMapper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -470,7 +470,19 @@ def getFilter(datasetType, pythonType, location, dataId):
# TODO: deprecate in DM-27177, remove in DM-27811
def getFilterLabel(datasetType, pythonType, location, dataId):
fitsReader = afwImage.ExposureFitsReader(location.getLocationsWithRoot()[0])
return fitsReader.readFilterLabel()
storedFilter = fitsReader.readFilterLabel()

# Apply standardization used by full Exposure
try:
# mapping is local to enclosing scope
idFilter = mapping.need(['filter'], dataId)['filter']
except dafPersist.NoResults:
idFilter = None
bestFilter = self._getBestFilter(storedFilter, idFilter)
if bestFilter is not None:
return bestFilter
else:
return storedFilter

setMethods("filterLabel", bypassImpl=getFilterLabel)

Expand Down Expand Up @@ -1060,6 +1072,55 @@ def _resolveFilters(definitions, idFilter, filterLabel):
matches.intersection_update(definitions.findAll(filterLabel.bandLabel))
return matches

def _getBestFilter(self, storedLabel, idFilter):
"""Estimate the most complete filter information consistent with the
file or registry.
Parameters
----------
storedLabel : `lsst.afw.image.FilterLabel` or `None`
The filter previously stored in the file.
idFilter : `str` or `None`
The filter implied by the data ID, if any.
Returns
-------
bestFitler : `lsst.afw.image.FilterLabel` or `None`
The complete filter to describe the dataset. May be equal to
``storedLabel``. `None` if no recommendation can be generated.
"""
try:
# getGen3Instrument returns class; need to construct it.
filterDefinitions = self.getGen3Instrument()().filterDefinitions
except NotImplementedError:
filterDefinitions = None

if filterDefinitions is not None:
definitions = self._resolveFilters(filterDefinitions, idFilter, storedLabel)
self.log.debug("Matching filters for id=%r and label=%r are %s.",
idFilter, storedLabel, definitions)
if len(definitions) == 1:
newLabel = list(definitions)[0].makeFilterLabel()
return newLabel
elif definitions:
self.log.warn("Multiple matches for filter %r with data ID %r.", storedLabel, idFilter)
# Can we at least add a band?
# Never expect multiple definitions with same physical filter.
bands = {d.band for d in definitions} # None counts as separate result!
if len(bands) == 1 and storedLabel is None:
band = list(bands)[0]
return afwImage.FilterLabel(band=band)
else:
return None
else:
# Unknown filter, nothing to be done.
self.log.warn("Cannot reconcile filter %r with data ID %r.", storedLabel, idFilter)
return None

# Not practical to recommend a FilterLabel without filterDefinitions

return None

def _setFilter(self, mapping, item, dataId):
"""Set the filter information in an Exposure.
Expand All @@ -1081,49 +1142,29 @@ def _setFilter(self, mapping, item, dataId):
or isinstance(item, afwImage.ExposureF) or isinstance(item, afwImage.ExposureD)):
return

try:
# getGen3Instrument returns class; need to construct it.
filterDefinitions = self.getGen3Instrument()().filterDefinitions
except NotImplementedError:
filterDefinitions = None
itemFilter = item.getFilterLabel() # may be None
try:
idFilter = mapping.need(['filter'], dataId)['filter']
except dafPersist.NoResults:
idFilter = None

if filterDefinitions is not None:
definitions = self._resolveFilters(filterDefinitions, idFilter, itemFilter)
self.log.debug("Matching filters for id=%r and label=%r are %s.",
idFilter, itemFilter, definitions)
if len(definitions) == 1:
newLabel = list(definitions)[0].makeFilterLabel()
if newLabel != itemFilter:
item.setFilterLabel(newLabel)
elif definitions:
self.log.warn("Multiple matches for filter %r with data ID %r.", itemFilter, idFilter)
# Can we at least add a band?
# Never expect multiple definitions with same physical filter.
bands = {d.band for d in definitions} # None counts as separate result!
if len(bands) == 1 and itemFilter is None:
band = list(bands)[0]
item.setFilterLabel(afwImage.FilterLabel(band=band))
else:
# Unknown filter, nothing to be done.
self.log.warn("Cannot reconcile filter %r with data ID %r.", itemFilter, idFilter)
else:
if itemFilter is None:
# Old Filter cleanup, without the benefit of FilterDefinition
if self.filters is not None and idFilter in self.filters:
idFilter = self.filters[idFilter]
try:
# TODO: remove in DM-27177; at that point may not be able
# to process IDs without FilterDefinition.
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=FutureWarning)
item.setFilter(afwImage.Filter(idFilter))
except pexExcept.NotFoundError:
self.log.warn("Filter %s not defined. Set to UNKNOWN.", idFilter)
bestFilter = self._getBestFilter(itemFilter, idFilter)
if bestFilter is not None:
if bestFilter != itemFilter:
item.setFilterLabel(bestFilter)
# Already using bestFilter, avoid unnecessary edits
elif itemFilter is None:
# Old Filter cleanup, without the benefit of FilterDefinition
if self.filters is not None and idFilter in self.filters:
idFilter = self.filters[idFilter]
try:
# TODO: remove in DM-27177; at that point may not be able
# to process IDs without FilterDefinition.
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=FutureWarning)
item.setFilter(afwImage.Filter(idFilter))
except pexExcept.NotFoundError:
self.log.warn("Filter %s not defined. Set to UNKNOWN.", idFilter)

def _standardizeExposure(self, mapping, item, dataId, filter=True,
trimmed=True, setVisitInfo=True):
Expand Down
6 changes: 6 additions & 0 deletions tests/MinMapper2.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,12 @@ exposures:
persistable: "ImageU"
level: "Ccd"
tables: "raw"
someExp:
template: "bar-%(ccd)02d.fits"
python: "lsst.afw.image.ExposureF"
persistable: "ImageF"
level: "Ccd"
tables: "raw"

images:
some:
Expand Down
89 changes: 54 additions & 35 deletions tests/test_cameraMapper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,11 @@ def filterDefinitions(self):
lsst.obs.base.FilterDefinition(physical_filter="g.MP9401", band="g", lambdaEff=487),
lsst.obs.base.FilterDefinition(physical_filter="r.MP9601", band="r", alias={"old-r"},
lambdaEff=628),
lsst.obs.base.FilterDefinition(physical_filter="i.MP9701", band="i", lambdaEff=778),
lsst.obs.base.FilterDefinition(physical_filter="i.MP9701", band="i", alias={"old-i"},
lambdaEff=778),
lsst.obs.base.FilterDefinition(physical_filter="z.MP9801", band="z", lambdaEff=1170),
# afw_name is so special-cased that only a real example will work
lsst.obs.base.FilterDefinition(physical_filter="HSC-R2", band="r", afw_name="r2", lambdaEff=623),
lsst.obs.base.FilterDefinition(physical_filter="HSC-I2", band="i", afw_name="i2", lambdaEff=623),
)

@classmethod
Expand Down Expand Up @@ -239,6 +240,10 @@ def testGetDatasetTypes(self):
"someGz", "someGz_filename", "someFz", "someFz_filename", "someGz_md",
"someFz_sub", "someFz_md", "someGz_sub",
"someGz_bbox", "someFz_bbox", "some_bbox", "other_bbox",
"someExp", "someExp_filename", "someExp_md", "someExp_sub",
"someExp_bbox", "someExp_filterLabel", "someExp_photoCalib",
"someExp_visitInfo", "someExp_detector", "someExp_filter",
"someExp_header_wcs", "someExp_wcs",
])
self.assertEqual(set(mapper.getDatasetTypes()),
set(expectedTypes))
Expand Down Expand Up @@ -330,6 +335,20 @@ def testImage(self):
self.assertEqual(image.getHeight(), 400)
self.assertEqual(image.getWidth(), 300)

def testFilter(self):
"""Test that the same (patched) filter is returned through all Butler
retrieval paths.
"""
mapper = MinMapper2(root=ROOT)

butler = dafPersist.ButlerFactory(mapper=mapper).create()
image = butler.get("someExp", ccd=35)
filter = butler.get("someExp_filterLabel", ccd=35)
# Test only valid with a complete filter
self.assertEqual(image.getFilterLabel(), afwImage.FilterLabel(band="r", physical="r.MP9601"))
# Datasets should give consistent answers
self.assertEqual(filter, image.getFilterLabel())

def testDetector(self):
mapper = MinMapper2(root=ROOT)
butler = dafPersist.ButlerFactory(mapper=mapper).create()
Expand Down Expand Up @@ -399,41 +418,41 @@ def testStandardize(self):
def testStandardizeFiltersFilterDefs(self):
testLabels = [
(None, None),
(afwImage.FilterLabel(band="r", physical="r.MP9601"),
afwImage.FilterLabel(band="r", physical="r.MP9601")),
(afwImage.FilterLabel(band="r"), afwImage.FilterLabel(band="r", physical="r.MP9601")),
(afwImage.FilterLabel(physical="r.MP9601"),
afwImage.FilterLabel(band="r", physical="r.MP9601")),
(afwImage.FilterLabel(band="r", physical="old-r"),
afwImage.FilterLabel(band="r", physical="r.MP9601")),
(afwImage.FilterLabel(physical="old-r"),
afwImage.FilterLabel(band="r", physical="r.MP9601")),
(afwImage.FilterLabel(physical="r2"), afwImage.FilterLabel(band="r", physical="HSC-R2")),
(afwImage.FilterLabel(band="i", physical="i.MP9701"),
afwImage.FilterLabel(band="i", physical="i.MP9701")),
(afwImage.FilterLabel(band="i"), afwImage.FilterLabel(band="r", physical="i.MP9701")),
(afwImage.FilterLabel(physical="i.MP9701"),
afwImage.FilterLabel(band="i", physical="i.MP9701")),
(afwImage.FilterLabel(band="i", physical="old-i"),
afwImage.FilterLabel(band="i", physical="i.MP9701")),
(afwImage.FilterLabel(physical="old-i"),
afwImage.FilterLabel(band="i", physical="i.MP9701")),
(afwImage.FilterLabel(physical="i2"), afwImage.FilterLabel(band="i", physical="HSC-I2")),
]
testIds = [{"visit": 12345, "ccd": 42, "filter": f} for f in {
"r", "r.MP9601", "old-r", "r2",
"i", "i.MP9701", "old-i", "i2",
}]
testData = []
# Resolve special combinations where the expected output is different
for input, corrected in testLabels:
for dataId in testIds:
if input is None:
if dataId["filter"] == "r":
data = (input, dataId, afwImage.FilterLabel(band="r"))
elif dataId["filter"] == "r2":
data = (input, dataId, afwImage.FilterLabel(band="r", physical="HSC-R2"))
if dataId["filter"] == "i":
data = (input, dataId, afwImage.FilterLabel(band="i"))
elif dataId["filter"] == "i2":
data = (input, dataId, afwImage.FilterLabel(band="i", physical="HSC-I2"))
else:
data = (input, dataId, afwImage.FilterLabel(band="r", physical="r.MP9601"))
elif input == afwImage.FilterLabel(band="r"):
if dataId["filter"] == "r":
# There are two "r" filters, can't tell which
data = (input, dataId, afwImage.FilterLabel(band="i", physical="i.MP9701"))
elif input == afwImage.FilterLabel(band="i"):
if dataId["filter"] == "i":
# There are two "i" filters, can't tell which
data = (input, dataId, input)
elif dataId["filter"] == "r2":
data = (input, dataId, afwImage.FilterLabel(band="r", physical="HSC-R2"))
elif corrected.physicalLabel == "HSC-R2" and dataId["filter"] in ("r.MP9601", "old-r"):
elif dataId["filter"] == "i2":
data = (input, dataId, afwImage.FilterLabel(band="i", physical="HSC-I2"))
elif corrected.physicalLabel == "HSC-I2" and dataId["filter"] in ("i.MP9701", "old-i"):
# Contradictory inputs, leave as-is
data = (input, dataId, input)
elif corrected.physicalLabel == "r.MP9601" and dataId["filter"] == "r2":
elif corrected.physicalLabel == "i.MP9701" and dataId["filter"] == "i2":
# Contradictory inputs, leave as-is
data = (input, dataId, input)
else:
Expand All @@ -450,29 +469,29 @@ def testStandardizeFiltersFilterDefs(self):
def testStandardizeFiltersFilterNoDefs(self):
testLabels = [
None,
afwImage.FilterLabel(band="r", physical="r.MP9601"),
afwImage.FilterLabel(band="r"),
afwImage.FilterLabel(physical="r.MP9601"),
afwImage.FilterLabel(band="r", physical="old-r"),
afwImage.FilterLabel(physical="old-r"),
afwImage.FilterLabel(physical="r2"),
afwImage.FilterLabel(band="i", physical="i.MP9701"),
afwImage.FilterLabel(band="i"),
afwImage.FilterLabel(physical="i.MP9701"),
afwImage.FilterLabel(band="i", physical="old-i"),
afwImage.FilterLabel(physical="old-i"),
afwImage.FilterLabel(physical="i2"),
]
testIds = [{"visit": 12345, "ccd": 42, "filter": f} for f in {
"r", "r.MP9601", "old-r", "r2",
"i", "i.MP9701", "old-i", "i2",
}]
testData = []
# Resolve special combinations where the expected output is different
for input in testLabels:
for dataId in testIds:
if input is None:
# Can still get some filter info out of the Filter registry
if dataId["filter"] == "r2":
if dataId["filter"] == "i2":
data = (input, dataId,
afwImage.FilterLabel(band="r", physical="HSC-R2"))
afwImage.FilterLabel(band="i", physical="HSC-I2"))
else:
# Data ID maps to filter(s) with aliases; can't
# unambiguously determine physical filter.
data = (input, dataId, afwImage.FilterLabel(band="r"))
data = (input, dataId, afwImage.FilterLabel(band="i"))
else:
data = (input, dataId, input)
testData.append(data)
Expand Down

0 comments on commit 6948e2c

Please sign in to comment.