Standardize the linearization code

Linearize using lsst.ip.isr.LinearizeSquared
(linearization type = "Squared"), retrieving it from the camera mapper
(though the coefficients come from the amp info table, as before).
Move suspect pixel masking to its standard location.
Add a config parameter to disable suspect pixel masking
(it used to be controlled by doLinearize).
Remove the existing "linearize" method, since it was no longer used.
Update amp info catalogs to set linearization type to "Squared"
if linearization is wanted, and "None" if not (e.g. suprimecam
and those HSC CCDs that had all zeros for their squared coefficient).

Here is the log of conversions:
hsc/camera (112 files)
0_00: Squared
0_01: Squared
0_02: Squared
0_03: Squared
0_04: Squared
0_05: Squared
0_06: Squared
0_07: Squared
0_08: Squared
0_09: Squared
0_10: Squared
0_11: Squared
0_12: Squared
0_13: Squared
0_14: Squared
0_15: Squared
0_16: Squared
0_17: Squared
0_18: Squared
0_19: Squared
0_20: Squared
0_21: Squared
0_22: Squared
0_23: Squared
0_24: Squared
0_25: Squared
0_26: Squared
0_27: Squared
0_28: Squared
0_29: Squared
0_30: Squared
0_31: Squared
0_32: Squared
0_33: Squared
0_34: Squared
0_35: Squared
0_36: Squared
0_37: Squared
0_38: Squared
0_39: Squared
0_40: Squared
0_41: Squared
0_42: Squared
0_43: Squared
0_44: Squared
0_45: Squared
0_46: Squared
0_47: Squared
0_48: None
0_51: None
0_52: None
0_53: Squared
0_54: Squared
0_55: Squared
0_56: Squared
0_57: None
1_00: Squared
1_01: Squared
1_02: Squared
1_03: Squared
1_04: Squared
1_05: Squared
1_06: Squared
1_07: Squared
1_08: Squared
1_09: Squared
1_10: Squared
1_11: Squared
1_12: Squared
1_13: Squared
1_14: Squared
1_15: Squared
1_16: Squared
1_17: Squared
1_18: Squared
1_19: Squared
1_20: Squared
1_21: Squared
1_22: Squared
1_23: Squared
1_24: Squared
1_25: Squared
1_26: Squared
1_27: Squared
1_28: Squared
1_29: Squared
1_30: Squared
1_31: Squared
1_32: Squared
1_33: Squared
1_34: Squared
1_35: Squared
1_36: Squared
1_37: Squared
1_38: Squared
1_39: Squared
1_40: Squared
1_41: Squared
1_42: Squared
1_43: Squared
1_44: Squared
1_45: Squared
1_46: Squared
1_47: Squared
1_48: None
1_51: None
1_52: None
1_53: Squared
1_54: Squared
1_55: Squared
1_56: Squared
1_57: None

suprimecam/camera (10 files)
Chihiro: None
Clarisse: None
Fio: None
Kiki: None
Nausicaa: None
Ponyo: None
San: None
Satsuki: None
Sheeta: None
Sophie: None

suprimecam/mit_camera (10 files)
si001s: None
si002s: None
si005s: None
si006s: None
w4c5: None
w67c1: None
w6c1: None
w7c3: None
w93c2: None
w9c2: None

python/lsst/obs/hsc/hscMapper.py

@@ -4,10 +4,12 @@
 
 import lsst.daf.base as dafBase
 from lsst.daf.butlerUtils import CameraMapper
+from lsst.daf.persistence import ButlerLocation
 import lsst.afw.image.utils as afwImageUtils
 import lsst.afw.image as afwImage
 import lsst.afw.math as afwMath
 import lsst.afw.geom as afwGeom
+from lsst.ip.isr import LinearizeSquared
 import lsst.pex.policy as pexPolicy
 
 class HscMapper(CameraMapper):
@@ -27,6 +29,8 @@ def __init__(self, **kwargs):
 
         super(HscMapper, self).__init__(policy, policyFile.getRepositoryPath(), **kwargs)
 
+        self._linearize = LinearizeSquared()
+
         # Ensure each dataset type of interest knows about the full range of keys available from the registry
         keys = {'field': str,
                 'visit': int,
@@ -255,6 +259,27 @@ def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
         """How many bits are required for the maximum exposure ID"""
         return 32 # just a guess, but this leaves plenty of space for sources
 
+    def map_linearize(self, dataId, write=False):
+        """Map a linearizer."""
+        if self._linearize is None:
+            raise RuntimeError("No linearizer available.")
+        actualId = self._transformId(dataId)
+        return ButlerLocation(
+            pythonType = "lsst.ip.isr.LinearizeSquared",
+            cppType = "Config",
+            storageName = "PickleStorage",
+            locationList = "ignored",
+            dataId = actualId,
+            mapper = self,
+        )
+
+    def bypass_linearize(self, datasetType, pythonType, butlerLocation, dataId):
+        """Return the linearizer.
+        """
+        if self._linearize is None:
+            raise RuntimeError("No linearizer available.")
+        return self._linearize
+
     def _computeCoaddExposureId(self, dataId, singleFilter):
         """Compute the 64-bit (long) identifier for a coadd.
 

python/lsst/obs/subaru/isr.py

@@ -86,6 +86,7 @@ def run(self, exposure):
 class SubaruIsrConfig(IsrTask.ConfigClass):
     qa = pexConfig.ConfigField(doc="QA-related config options", dtype=QaConfig)
     doSaturation = pexConfig.Field(doc="Mask saturated pixels?", dtype=bool, default=True)
+    doSuspect = pexConfig.Field(doc="Mask suspect pixels?", dtype=bool, default=True)
     doWidenSaturationTrails = pexConfig.Field(doc="Widen bleed trails based on their width?",
                                               dtype=bool, default=True)
     doOverscan = pexConfig.Field(doc="Do overscan subtraction?", dtype=bool, default=True)
@@ -102,12 +103,6 @@ class SubaruIsrConfig(IsrTask.ConfigClass):
         doc = "Correct for crosstalk",
         default = True,
     )
-    doLinearize = pexConfig.Field(
-        dtype = bool,
-        doc = "Correct for nonlinearity of the detector's response and set suspect level " \
-                "(ignored if coefficients are 0.0)",
-        default = True,
-    )
     doApplyGains = pexConfig.Field(
         dtype = bool,
         doc = """Correct the amplifiers for their gains
@@ -246,6 +241,8 @@ def runDataRef(self, sensorRef):
             self.measureOverscan(ccdExposure, amp)
             if self.config.doSaturation and not badAmp:
                 self.saturationDetection(ccdExposure, amp)
+            if self.config.doSuspect and not badAmp:
+                self.suspectDetection(ccdExposure, amp)
             if self.config.doOverscan and not badAmp:
                 ampImage = afwImage.MaskedImageF(ccdExposure.getMaskedImage(), amp.getRawDataBBox(),
                                                  afwImage.PARENT)
@@ -285,8 +282,10 @@ def runDataRef(self, sensorRef):
         if self.config.doBias:
             biasExposure = self.getIsrExposure(sensorRef, "bias")
             self.biasCorrection(ccdExposure, biasExposure)
-        if self.config.doLinearize:
-            self.linearize(ccdExposure)
+        if self.doLinearize(ccd):
+            self.log.log(self.log.INFO, "Applying linearity corrections to Ccd %s" % (ccd.getId()))
+            linearizer = sensorRef.get("linearize")
+            linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
         if self.config.doCrosstalk:
             self.crosstalk.run(ccdExposure)
 
@@ -527,58 +526,6 @@ def guider(self, exposure):
             "Guider shadow trimming is enabled but no generic implementation is present"
             )
 
-    def linearize(self, exposure):
-        """Correct for non-linearity
-
-        @param exposure Exposure to process
-        """
-        assert exposure, "No exposure provided"
-
-        ccd = exposure.getDetector()
-
-        if ccd.getSerial() in map(str, range(104, 112)):
-            self.log.warn("Skipping linearity correction for focus CCD %s: no coefficients available" %
-                          (ccd.getSerial(),))
-            return
-
-        linearized = False              # did we apply linearity corrections?
-        for amp in ccd:
-            linearityCoefficient = amp.getLinearityCoeffs()[0]
-            linearityThreshold = amp.getLinearityCoeffs()[1]
-            linearityMaxCorrectable = amp.getSuspectLevel()
-            linearityType = amp.getLinearityType()
-
-            ampImage = afwImage.MaskedImageF(exposure.getMaskedImage(), amp.getBBox(),
-                                                 afwImage.PARENT)
-
-            setSuspectPixels = not math.isnan(linearityMaxCorrectable)
-
-            if not setSuspectPixels and linearityCoefficient == 0.0:
-                continue                # nothing to do
-
-            linearized = True
-            #
-            # We may have a max correctable level even if we make no attempt to correct
-            #
-            if setSuspectPixels:
-                afwDetection.FootprintSet(ampImage,
-                                          afwDetection.Threshold(linearityMaxCorrectable), "SUSPECT")
-
-            if linearityType == 'PROPORTIONAL':
-                if linearityThreshold != 0.0:
-                    raise RuntimeError(
-                        ("The threshold for PROPORTIONAL linearity corrections must be 0; saw %g" +
-                         " for ccd %s amp %s") % (linearityThreshold, ccd.getId(), amp.getName()))
-
-                ampArr = ampImage.getImage().getArray()
-                ampArr *= 1.0 + linearityCoefficient*ampArr
-            else:
-                raise NotImplementedError("Unimplemented linearity type: %d", linearityType)
-
-        if linearized:
-            self.log.log(self.log.INFO, "Applying linearity corrections to Ccd %s" % (ccd.getId()))
-
-
     def flatCorrection(self, exposure, flatExposure):
         """Apply flat correction in-place
 

python/lsst/obs/suprimecam/suprimecamMapper.py

@@ -6,7 +6,9 @@
 import lsst.afw.geom as afwGeom
 import lsst.afw.image.utils as afwImageUtils
 
+from lsst.daf.persistence import ButlerLocation
 from lsst.daf.butlerUtils import CameraMapper
+from lsst.ip.isr import LinearizeSquared
 import lsst.pex.policy as pexPolicy
 
 class SuprimecamMapperBase(CameraMapper):
@@ -15,6 +17,8 @@ class SuprimecamMapperBase(CameraMapper):
     def __init__(self, *args, **kwargs):
         super(SuprimecamMapperBase, self).__init__(*args, **kwargs)
 
+        self._linearize = LinearizeSquared()
+
         # Ensure each dataset type of interest knows about the full range of keys available from the registry
         keys = {'field': str,
                 'visit': int,
@@ -217,6 +221,27 @@ def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
     def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
         return 32 # not really, but this leaves plenty of space for sources
 
+    def map_linearize(self, dataId, write=False):
+        """Map a linearizer."""
+        if self._linearize is None:
+            raise RuntimeError("No linearizer available.")
+        actualId = self._transformId(dataId)
+        return ButlerLocation(
+            pythonType = "lsst.ip.isr.LinearizeSquared",
+            cppType = "Config",
+            storageName = "PickleStorage",
+            locationList = "ignored",
+            dataId = actualId,
+            mapper = self,
+        )
+
+    def bypass_linearize(self, datasetType, pythonType, butlerLocation, dataId):
+        """Return the linearizer.
+        """
+        if self._linearize is None:
+            raise RuntimeError("No linearizer available.")
+        return self._linearize
+
     def _computeStackExposureId(self, dataId):
         """Compute the 64-bit (long) identifier for a Stack exposure.