Skip to content

Commit

Permalink
Merge branch 'tickets/DM-26545' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
@czwa
czwa committed Oct 23, 2020
2 parents 7966b28 + 943c655 commit 9bbb7f7
Show file tree
Hide file tree
Showing 3 changed files with 177 additions and 50 deletions.
116 changes: 92 additions & 24 deletions python/lsst/ip/isr/calibType.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,17 +60,18 @@ class IsrCalib(abc.ABC):
_SCHEMA = 'NO SCHEMA'
_VERSION = 0

def __init__(self, camera=None, detector=None, detectorName=None, detectorId=None, log=None, **kwargs):
def __init__(self, camera=None, detector=None, log=None, **kwargs):
self._instrument = None
self._raftName = None
self._slotName = None
self._detectorName = detectorName
self._detectorName = None
self._detectorSerial = None
self._detectorId = detectorId
self._detectorId = None
self._filter = None
self._calibId = None
self._metadata = PropertyList()
self.setMetadata(PropertyList())
self.calibInfoFromDict(kwargs)

# Define the required attributes for this calibration.
self.requiredAttributes = set(['_OBSTYPE', '_SCHEMA', '_VERSION'])
Expand All @@ -82,7 +83,7 @@ def __init__(self, camera=None, detector=None, detectorName=None, detectorId=Non

if detector:
self.fromDetector(detector)
self.updateMetadata(camera=camera, detector=detector, setDate=False)
self.updateMetadata(camera=camera, detector=detector)

def __str__(self):
return f"{self.__class__.__name__}(obstype={self._OBSTYPE}, detector={self._detectorName}, )"
Expand Down Expand Up @@ -139,8 +140,13 @@ def setMetadata(self, metadata):
self._metadata[self._OBSTYPE + "_SCHEMA"] = self._SCHEMA
self._metadata[self._OBSTYPE + "_VERSION"] = self._VERSION

if isinstance(metadata, dict):
self.calibInfoFromDict(metadata)
elif isinstance(metadata, PropertyList):
self.calibInfoFromDict(metadata.toDict())

def updateMetadata(self, camera=None, detector=None, filterName=None,
setCalibId=False, setDate=False,
setCalibId=False, setCalibInfo=False, setDate=False,
**kwargs):
"""Update metadata keywords with new values.
Expand All @@ -162,6 +168,9 @@ def updateMetadata(self, camera=None, detector=None, filterName=None,
mdOriginal = self.getMetadata()
mdSupplemental = dict()

if setCalibInfo:
self.calibInfoFromDict(kwargs)

if camera:
self._instrument = camera.getName()

Expand All @@ -178,20 +187,24 @@ def updateMetadata(self, camera=None, detector=None, filterName=None,
# then this will hold the abstract filter.
self._filter = filterName

if setDate:
date = datetime.datetime.now()
mdSupplemental['CALIBDATE'] = date.isoformat()
mdSupplemental['CALIB_CREATION_DATE'] = date.date().isoformat()
mdSupplemental['CALIB_CREATION_TIME'] = date.time().isoformat()

if setCalibId:
values = []
values.append(f"instrument={self._instrument}") if self._instrument else None
values.append(f"raftName={self._raftName}") if self._raftName else None
values.append(f"detectorName={self._detectorName}") if self._detectorName else None
values.append(f"detector={self.detector}") if self._detector else None
values.append(f"detector={self._detectorId}") if self._detectorId else None
values.append(f"filter={self._filter}") if self._filter else None
self._calibId = " ".join(values)

if setDate:
date = datetime.datetime.now()
mdSupplemental['CALIBDATE'] = date.isoformat()
mdSupplemental['CALIB_CREATION_DATE'] = date.date().isoformat()
mdSupplemental['CALIB_CREATION_TIME'] = date.time().isoformat()
calibDate = mdOriginal.get('CALIBDATE', mdSupplemental.get('CALIBDATE', None))
values.append(f"calibDate={calibDate}") if calibDate else None

self._calibId = " ".join(values)

self._metadata["INSTRUME"] = self._instrument if self._instrument else None
self._metadata["RAFTNAME"] = self._raftName if self._raftName else None
Expand All @@ -205,6 +218,60 @@ def updateMetadata(self, camera=None, detector=None, filterName=None,
mdSupplemental.update(kwargs)
mdOriginal.update(mdSupplemental)

def calibInfoFromDict(self, dictionary):
"""Handle common keywords.
This isn't an ideal solution, but until all calibrations
expect to find everything in the metadata, they still need to
search through dictionaries.
Parameters
----------
dictionary : `dict` or `lsst.daf.base.PropertyList`
Source for the common keywords.
Raises
------
RuntimeError :
Raised if the dictionary does not match the expected OBSTYPE.
"""

def search(haystack, needles):
"""Search dictionary 'haystack' for an entry in 'needles'
"""
test = [haystack.get(x) for x in needles]
test = set([x for x in test if x is not None])
if len(test) == 0:
if 'metadata' in haystack:
return search(haystack['metadata'], needles)
else:
return None
elif len(test) == 1:
value = list(test)[0]
if value == '':
return None
else:
return value
else:
raise ValueError(f"Too many values found: {len(test)} {test} {needles}")

if 'metadata' in dictionary:
metadata = dictionary['metadata']

if self._OBSTYPE != metadata['OBSTYPE']:
raise RuntimeError(f"Incorrect calibration supplied. Expected {self._OBSTYPE}, "
f"found {metadata['OBSTYPE']}")

self._instrument = search(dictionary, ['INSTRUME', 'instrument'])
self._raftName = search(dictionary, ['RAFTNAME'])
self._slotName = search(dictionary, ['SLOTNAME'])
self._detectorId = search(dictionary, ['DETECTOR', 'detectorId'])
self._detectorName = search(dictionary, ['DET_NAME', 'DETECTOR_NAME', 'detectorName'])
self._detectorSerial = search(dictionary, ['DET_SER', 'DETECTOR_SERIAL', 'detectorSerial'])
self._filter = search(dictionary, ['FILTER', 'filterName'])
self._calibId = search(dictionary, ['CALIB_ID'])

@classmethod
def readText(cls, filename):
"""Read calibration representation from a yaml/ecsv file.
Expand Down Expand Up @@ -317,6 +384,11 @@ def readFits(cls, filename):
except Exception:
keepTrying = False

for table in tableList:
for k, v in table.meta.items():
if isinstance(v, fits.card.Undefined):
table.meta[k] = None

return cls.fromTable(tableList)

def writeFits(self, filename):
Expand Down Expand Up @@ -560,9 +632,6 @@ def fromTable(cls, tableList):
metadata = table.meta
inDict = dict()
inDict['metadata'] = metadata
inDict['detectorName'] = metadata.get('DET_NAME', None)
inDict['detectorSerial'] = metadata.get('DET_SER', None)
inDict['instrument'] = metadata.get('INSTRUME', None)
inDict['calibType'] = metadata['calibType']
inDict['dimensions'] = set()
inDict['dataIdList'] = list()
Expand Down Expand Up @@ -596,17 +665,15 @@ def fromDict(cls, dictionary):
The provenance defined in the tables.
"""
calib = cls()
calib.updateMetadata(setDate=False, **dictionary['metadata'])
if calib._OBSTYPE != dictionary['metadata']['OBSTYPE']:
raise RuntimeError(f"Incorrect calibration supplied. Expected {calib._OBSTYPE}, "
f"found {dictionary['metadata']['OBSTYPE']}")

calib.setMetadata(dictionary['metadata'])

# These properties should be in the metadata, but occasionally
# are found in the dictionary itself. Check both places,
# ending with `None` if neither contains the information.
calib._detectorName = dictionary.get('detectorName',
dictionary['metadata'].get('DET_NAME', None))
calib._detectorSerial = dictionary.get('detectorSerial',
dictionary['metadata'].get('DET_SER', None))
calib._instrument = dictionary.get('instrument',
dictionary['metadata'].get('INSTRUME', None))
calib.calibType = dictionary['calibType']
calib.dimensions = set(dictionary['dimensions'])
calib.dataIdList = dictionary['dataIdList']
Expand All @@ -622,14 +689,15 @@ def toDict(self):
dictionary : `dict`
Dictionary of provenance.
"""
self.updateMetadata(setDate=True)
self.updateMetadata()

outDict = {}

metadata = self.getMetadata()
outDict['metadata'] = metadata
outDict['detectorName'] = self._detectorName
outDict['detectorSerial'] = self._detectorSerial
outDict['detectorId'] = self._detectorId
outDict['instrument'] = self._instrument
outDict['calibType'] = self.calibType
outDict['dimensions'] = list(self.dimensions)
Expand All @@ -650,7 +718,7 @@ def toTable(self):
"""
tableList = []
self.updateMetadata(setDate=True)
self.updateMetadata()
catalog = Table(rows=self.dataIdList,
names=self.dimensions)
filteredMetadata = {k: v for k, v in self.getMetadata().toDict().items() if v is not None}
Expand Down

0 comments on commit 9bbb7f7

Please sign in to comment.