Merge pull request #120 from lsst/tickets/DM-22659

Fix bitrot for using new style brighter-fatter kernels

python/lsst/ip/isr/isrFunctions.py

@@ -647,7 +647,7 @@ def overscanCorrection(ampMaskedImage, overscanImage, fitType='MEDIAN', order=1,
     return Struct(imageFit=offImage, overscanFit=overscanFit, overscanImage=overscanImage)
 
 
-def brighterFatterCorrection(exposure, kernel, maxIter, threshold, applyGain):
+def brighterFatterCorrection(exposure, kernel, maxIter, threshold, applyGain, gains=None):
     """Apply brighter fatter correction in place for the image.
 
     Parameters
@@ -665,6 +665,9 @@ def brighterFatterCorrection(exposure, kernel, maxIter, threshold, applyGain):
     applyGain : `Bool`
         If True, then the exposure values are scaled by the gain prior
         to correction.
+    gains : `dict` [`str`, `float`]
+        A dictionary, keyed by amplifier name, of the gains to use.
+        If gains is None, the nominal gains in the amplifier object are used.
 
     Returns
     -------
@@ -705,7 +708,7 @@ def brighterFatterCorrection(exposure, kernel, maxIter, threshold, applyGain):
     image = exposure.getMaskedImage().getImage()
 
     # The image needs to be units of electrons/holes
-    with gainContext(exposure, image, applyGain):
+    with gainContext(exposure, image, applyGain, gains):
 
         kLx = numpy.shape(kernel)[0]
         kLy = numpy.shape(kernel)[1]
@@ -767,7 +770,7 @@ def brighterFatterCorrection(exposure, kernel, maxIter, threshold, applyGain):
 
 
 @contextmanager
-def gainContext(exp, image, apply):
+def gainContext(exp, image, apply, gains=None):
     """Context manager that applies and removes gain.
 
     Parameters
@@ -778,17 +781,32 @@ def gainContext(exp, image, apply):
         Image to apply/remove gain.
     apply : `Bool`
         If True, apply and remove the amplifier gain.
+    gains : `dict` [`str`, `float`]
+        A dictionary, keyed by amplifier name, of the gains to use.
+        If gains is None, the nominal gains in the amplifier object are used.
 
     Yields
     ------
     exp : `lsst.afw.image.Exposure`
         Exposure with the gain applied.
     """
+    # check we have all of them if provided because mixing and matching would
+    # be a real mess
+    if gains and apply is True:
+        ampNames = [amp.getName() for amp in exp.getDetector()]
+        for ampName in ampNames:
+            if ampName not in gains.keys():
+                raise RuntimeError(f"Gains provided to gain context, but no entry found for amp {ampName}")
+
     if apply:
         ccd = exp.getDetector()
         for amp in ccd:
             sim = image.Factory(image, amp.getBBox())
-            sim *= amp.getGain()
+            if gains:
+                gain = gains[amp.getName()]
+            else:
+                gain = amp.getGain()
+            sim *= gain
 
     try:
         yield exp
@@ -797,7 +815,11 @@ def gainContext(exp, image, apply):
             ccd = exp.getDetector()
             for amp in ccd:
                 sim = image.Factory(image, amp.getBBox())
-                sim /= amp.getGain()
+                if gains:
+                    gain = gains[amp.getName()]
+                else:
+                    gain = amp.getGain()
+                sim /= gain
 
 
 def attachTransmissionCurve(exposure, opticsTransmission=None, filterTransmission=None,

python/lsst/ip/isr/isrTask.py

@@ -499,11 +499,6 @@ class IsrTaskConfig(pipeBase.PipelineTaskConfig,
             "DETECTOR": "One kernel per detector",
         }
     )
-    brighterFatterKernelFile = pexConfig.Field(
-        dtype=str,
-        default='',
-        doc="Kernel file used for the brighter fatter correction"
-    )
     brighterFatterMaxIter = pexConfig.Field(
         dtype=int,
         default=10,
@@ -825,6 +820,10 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs):
                 if not isinstance(inputs["defects"], Defects):
                     inputs["defects"] = Defects.fromTable(inputs["defects"])
 
+        # TODO: DM-22776 add retrieval for brighter-fatter kernel for Gen3
+        # if we can get HSC to use a new kernel, or just translate the old
+        # one to the new format we could drop the whole new/old style support
+
         # Broken: DM-17169
         # ci_hsc does not use crosstalkSources, as it's intra-CCD CT only.  This needs to be
         # fixed for non-HSC cameras in the future.
@@ -918,22 +917,31 @@ def readIsrData(self, dataRef, rawExposure):
                         if self.config.doFlat else None)
 
         brighterFatterKernel = None
+        brighterFatterGains = None
         if self.config.doBrighterFatter is True:
-
-            # Use the new-style cp_pipe version of the kernel is it exists.
             try:
+                # Use the new-style cp_pipe version of the kernel if it exists
+                # If using a new-style kernel, always use the self-consistent
+                # gains, i.e. the ones inside the kernel object itself
                 brighterFatterKernel = dataRef.get("brighterFatterKernel")
+                brighterFatterGains = brighterFatterKernel.gain
+                self.log.info("New style bright-fatter kernel (brighterFatterKernel) loaded")
             except NoResults:
-                # Fall back to the old-style numpy-ndarray style kernel if necessary.
-                try:
+                try:  # Fall back to the old-style numpy-ndarray style kernel if necessary.
                     brighterFatterKernel = dataRef.get("bfKernel")
+                    self.log.info("Old style bright-fatter kernel (np.array) loaded")
                 except NoResults:
                     brighterFatterKernel = None
             if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
-                # If the kernel is not an ndarray, it's the cp_pipe version, so extract the kernel for
-                # this detector, or raise an error.
+                # If the kernel is not an ndarray, it's the cp_pipe version
+                # so extract the kernel for this detector, or raise an error
                 if self.config.brighterFatterLevel == 'DETECTOR':
-                    brighterFatterKernel = brighterFatterKernel.kernel[ccd.getId()]
+                    if brighterFatterKernel.detectorKernel:
+                        brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
+                    elif brighterFatterKernel.detectorKernelFromAmpKernels:
+                        brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()]
+                    else:
+                        raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
                 else:
                     # TODO DM-15631 for implementing this
                     raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
@@ -978,6 +986,7 @@ def readIsrData(self, dataRef, rawExposure):
                                dark=darkExposure,
                                flat=flatExposure,
                                bfKernel=brighterFatterKernel,
+                               bfGains=brighterFatterGains,
                                defects=defectList,
                                fringes=fringeStruct,
                                opticsTransmission=opticsTransmission,
@@ -990,8 +999,8 @@ def readIsrData(self, dataRef, rawExposure):
 
     @pipeBase.timeMethod
     def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSources=None,
-            dark=None, flat=None, bfKernel=None, defects=None, fringes=pipeBase.Struct(fringes=None),
-            opticsTransmission=None, filterTransmission=None,
+            dark=None, flat=None, bfKernel=None, bfGains=None, defects=None,
+            fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
             sensorTransmission=None, atmosphereTransmission=None,
             detectorNum=None, strayLightData=None, illumMaskedImage=None,
             isGen3=False,
@@ -1035,6 +1044,10 @@ def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSou
             Flat calibration frame.
         bfKernel : `numpy.ndarray`, optional
             Brighter-fatter kernel.
+        bfGains : `dict` of `float`, optional
+            Gains used to override the detector's nominal gains for the
+            brighter-fatter correction. A dict keyed by amplifier name for
+            the detector in question.
         defects : `lsst.meas.algorithms.Defects`, optional
             List of defects.
         fringes : `lsst.pipe.base.Struct`, optional
@@ -1296,8 +1309,8 @@ def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalkSou
             bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
                                                               self.config.brighterFatterMaxIter,
                                                               self.config.brighterFatterThreshold,
-                                                              self.config.brighterFatterApplyGain
-                                                              )
+                                                              self.config.brighterFatterApplyGain,
+                                                              bfGains)
             if bfResults[1] == self.config.brighterFatterMaxIter:
                 self.log.warn("Brighter fatter correction did not converge, final difference %f.",
                               bfResults[0])