Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

DM-40128: Fix brighter-fatter loading errors #279

Merged
merged 2 commits into from
Sep 6, 2023
Merged

DM-40128: Fix brighter-fatter loading errors #279

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a1740ec
Simplify header provenance code.
czwa Jul 20, 2023
eb870b7
Clarify logic.
czwa Sep 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
101 changes: 54 additions & 47 deletions python/lsst/ip/isr/isrTask.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1027,6 +1027,9 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs):

detector = inputs['ccdExposure'].getDetector()

# This is use for header provenance.
additionalInputDates = {}

if self.config.doCrosstalk is True:
# Crosstalk sources need to be defined by the pipeline
# yaml if they exist.
Expand Down Expand Up @@ -1071,21 +1074,30 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs):

# Load the correct style of brighter-fatter kernel, and repack
# the information as a numpy array.
brighterFatterSource = None
if self.config.doBrighterFatter:
brighterFatterKernel = inputs.pop('newBFKernel', None)
if brighterFatterKernel is None:
# This type of kernel must be in (y, x) index
# ordering, as it used directly as the .array
# component of the afwImage kernel.
brighterFatterKernel = inputs.get('bfKernel', None)
brighterFatterSource = 'bfKernel'
additionalInputDates[brighterFatterSource] = self.extractCalibDate(brighterFatterKernel)

if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
if brighterFatterKernel is None:
# This was requested by the config, but none were found.
raise RuntimeError("No brighter-fatter kernel was supplied.")
elif not isinstance(brighterFatterKernel, numpy.ndarray):
# This is a ISR calib kernel. These kernels are
# generated in (x, y) index ordering, and need to be
# transposed to be used directly as the .array
# component of the afwImage kernel. This is done
# explicitly below when setting the ``bfKernel``
# input.
brighterFatterSource = 'newBFKernel'
additionalInputDates[brighterFatterSource] = self.extractCalibDate(brighterFatterKernel)

detName = detector.getName()
level = brighterFatterKernel.level

Expand Down Expand Up @@ -1128,16 +1140,30 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs):
if self.config.doHeaderProvenance:
# Add calibration provenanace info to header.
exposureMetadata = inputs['ccdExposure'].getMetadata()
for inputName in sorted(inputs.keys()):

# These inputs change name during this step. These should
# have matching entries in the additionalInputDates dict.
additionalInputs = []
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not that it matters much but couldn't you just set this to list(inputs.keys()) to start with and append to/extend it?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That would certainly work the same, but I think it would require some variable name and comment rewrites. I think in the long term, this can eventually go away if all cameras switch to IsrCalib style brighter-fatter kernels.

if self.config.doBrighterFatter:
additionalInputs.append(brighterFatterSource)

for inputName in sorted(list(inputs.keys()) + additionalInputs):
reference = getattr(inputRefs, inputName, None)
if reference is not None and hasattr(reference, "run"):
runKey = f"LSST CALIB RUN {inputName.upper()}"
runValue = reference.run
idKey = f"LSST CALIB UUID {inputName.upper()}"
idValue = str(reference.id)
dateKey = f"LSST CALIB DATE {inputName.upper()}"

if inputName in additionalInputDates:
dateValue = additionalInputDates[inputName]
else:
dateValue = self.extractCalibDate(inputs[inputName])

exposureMetadata[runKey] = runValue
exposureMetadata[idKey] = idValue
exposureMetadata[dateKey] = dateValue

outputs = self.run(**inputs)
butlerQC.put(outputs, outputRefs)
Expand Down Expand Up @@ -1320,51 +1346,32 @@ def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
if (self.config.doDeferredCharge and deferredChargeCalib is None):
raise RuntimeError("Must supply a deferred charge calibration if config.doDeferredCharge=True.")

if self.config.doHeaderProvenance:
# Inputs have been validated, so we can add their date
# information to the output header.
exposureMetadata = ccdExposure.getMetadata()
if self.config.doBias:
exposureMetadata["LSST CALIB DATE BIAS"] = self.extractCalibDate(bias)
self.compareCameraKeywords(exposureMetadata, bias, "bias")
if self.config.doBrighterFatter:
exposureMetadata["LSST CALIB DATE BFK"] = self.extractCalibDate(bfKernel)
self.compareCameraKeywords(exposureMetadata, bfKernel, "brighter-fatter")
if self.config.doCrosstalk:
exposureMetadata["LSST CALIB DATE CROSSTALK"] = self.extractCalibDate(crosstalk)
self.compareCameraKeywords(exposureMetadata, crosstalk, "crosstalk")
if self.config.doDark:
exposureMetadata["LSST CALIB DATE DARK"] = self.extractCalibDate(dark)
self.compareCameraKeywords(exposureMetadata, dark, "dark")
if self.config.doDefect:
exposureMetadata["LSST CALIB DATE DEFECTS"] = self.extractCalibDate(defects)
self.compareCameraKeywords(exposureMetadata, defects, "defects")
if self.config.doDeferredCharge:
exposureMetadata["LSST CALIB DATE CTI"] = self.extractCalibDate(deferredChargeCalib)
self.compareCameraKeywords(exposureMetadata, deferredChargeCalib, "CTI")
if self.config.doFlat:
exposureMetadata["LSST CALIB DATE FLAT"] = self.extractCalibDate(flat)
self.compareCameraKeywords(exposureMetadata, flat, "flat")
if (self.config.doFringe and physicalFilter in self.fringe.config.filters):
exposureMetadata["LSST CALIB DATE FRINGE"] = self.extractCalibDate(fringes.fringes)
self.compareCameraKeywords(exposureMetadata, fringes.fringes, "fringe")
if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters):
exposureMetadata["LSST CALIB DATE ILLUMINATION"] = self.extractCalibDate(illumMaskedImage)
self.compareCameraKeywords(exposureMetadata, illumMaskedImage, "illumination")
if self.doLinearize(ccd):
exposureMetadata["LSST CALIB DATE LINEARIZER"] = self.extractCalibDate(linearizer)
self.compareCameraKeywords(exposureMetadata, linearizer, "linearizer")
if self.config.usePtcGains or self.config.usePtcReadNoise:
exposureMetadata["LSST CALIB DATE PTC"] = self.extractCalibDate(ptc)
self.compareCameraKeywords(exposureMetadata, ptc, "PTC")
if self.config.doStrayLight:
exposureMetadata["LSST CALIB DATE STRAYLIGHT"] = self.extractCalibDate(strayLightData)
self.compareCameraKeywords(exposureMetadata, strayLightData, "straylight")
if self.config.doAttachTransmissionCurve:
exposureMetadata["LSST CALIB DATE OPTICS_TR"] = self.extractCalibDate(opticsTransmission)
exposureMetadata["LSST CALIB DATE FILTER_TR"] = self.extractCalibDate(filterTransmission)
exposureMetadata["LSST CALIB DATE SENSOR_TR"] = self.extractCalibDate(sensorTransmission)
exposureMetadata["LSST CALIB DATE ATMOSP_TR"] = self.extractCalibDate(atmosphereTransmission)
# Validate that the inputs match the exposure configuration.
exposureMetadata = ccdExposure.getMetadata()
if self.config.doBias:
self.compareCameraKeywords(exposureMetadata, bias, "bias")
if self.config.doBrighterFatter:
self.compareCameraKeywords(exposureMetadata, bfKernel, "brighter-fatter")
if self.config.doCrosstalk:
self.compareCameraKeywords(exposureMetadata, crosstalk, "crosstalk")
if self.config.doDark:
self.compareCameraKeywords(exposureMetadata, dark, "dark")
if self.config.doDefect:
self.compareCameraKeywords(exposureMetadata, defects, "defects")
if self.config.doDeferredCharge:
self.compareCameraKeywords(exposureMetadata, deferredChargeCalib, "CTI")
if self.config.doFlat:
self.compareCameraKeywords(exposureMetadata, flat, "flat")
if (self.config.doFringe and physicalFilter in self.fringe.config.filters):
self.compareCameraKeywords(exposureMetadata, fringes.fringes, "fringe")
if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters):
self.compareCameraKeywords(exposureMetadata, illumMaskedImage, "illumination")
if self.doLinearize(ccd):
self.compareCameraKeywords(exposureMetadata, linearizer, "linearizer")
if self.config.usePtcGains or self.config.usePtcReadNoise:
self.compareCameraKeywords(exposureMetadata, ptc, "PTC")
if self.config.doStrayLight:
self.compareCameraKeywords(exposureMetadata, strayLightData, "straylight")

# Begin ISR processing.
if self.config.doConvertIntToFloat:
Expand Down