Standardize the linearization code

Linearize using lsst.ip.isr.LinearizeSquared
(linearization type = "Squared"), retrieving it from the camera mapper
(though the coefficients come from the amp info table, as before).
Move suspect pixel masking to its standard location.
Add a config parameter to disable suspect pixel masking
(it used to be controlled by doLinearize).
Remove the existing "linearize" method, since it was no longer used.
Update amp info catalogs to set linearization type to "Squared"
if linearization is wanted, and "None" if not (e.g. suprimecam
and those HSC CCDs that had all zeros for their squared coefficient).

Here is the log of conversions:
hsc/camera
Converting 112 files
0_00.fits: 'Squared'
0_01.fits: 'Squared'
0_02.fits: 'Squared'
0_03.fits: 'Squared'
0_04.fits: 'Squared'
0_05.fits: 'Squared'
0_06.fits: 'Squared'
0_07.fits: 'Squared'
0_08.fits: 'Squared'
0_09.fits: 'Squared'
0_10.fits: 'Squared'
0_11.fits: 'Squared'
0_12.fits: 'Squared'
0_13.fits: 'Squared'
0_14.fits: 'Squared'
0_15.fits: 'Squared'
0_16.fits: 'Squared'
0_17.fits: 'Squared'
0_18.fits: 'Squared'
0_19.fits: 'Squared'
0_20.fits: 'Squared'
0_21.fits: 'Squared'
0_22.fits: 'Squared'
0_23.fits: 'Squared'
0_24.fits: 'Squared'
0_25.fits: 'Squared'
0_26.fits: 'Squared'
0_27.fits: 'Squared'
0_28.fits: 'Squared'
0_29.fits: 'Squared'
0_30.fits: 'Squared'
0_31.fits: 'Squared'
0_32.fits: 'Squared'
0_33.fits: 'Squared'
0_34.fits: 'Squared'
0_35.fits: 'Squared'
0_36.fits: 'Squared'
0_37.fits: 'Squared'
0_38.fits: 'Squared'
0_39.fits: 'Squared'
0_40.fits: 'Squared'
0_41.fits: 'Squared'
0_42.fits: 'Squared'
0_43.fits: 'Squared'
0_44.fits: 'Squared'
0_45.fits: 'Squared'
0_46.fits: 'Squared'
0_47.fits: 'Squared'
0_48.fits: 'None'
0_51.fits: 'None'
0_52.fits: 'None'
0_53.fits: 'Squared'
0_54.fits: 'Squared'
0_55.fits: 'Squared'
0_56.fits: 'Squared'
0_57.fits: 'None'
1_00.fits: 'Squared'
1_01.fits: 'Squared'
1_02.fits: 'Squared'
1_03.fits: 'Squared'
1_04.fits: 'Squared'
1_05.fits: 'Squared'
1_06.fits: 'Squared'
1_07.fits: 'Squared'
1_08.fits: 'Squared'
1_09.fits: 'Squared'
1_10.fits: 'Squared'
1_11.fits: 'Squared'
1_12.fits: 'Squared'
1_13.fits: 'Squared'
1_14.fits: 'Squared'
1_15.fits: 'Squared'
1_16.fits: 'Squared'
1_17.fits: 'Squared'
1_18.fits: 'Squared'
1_19.fits: 'Squared'
1_20.fits: 'Squared'
1_21.fits: 'Squared'
1_22.fits: 'Squared'
1_23.fits: 'Squared'
1_24.fits: 'Squared'
1_25.fits: 'Squared'
1_26.fits: 'Squared'
1_27.fits: 'Squared'
1_28.fits: 'Squared'
1_29.fits: 'Squared'
1_30.fits: 'Squared'
1_31.fits: 'Squared'
1_32.fits: 'Squared'
1_33.fits: 'Squared'
1_34.fits: 'Squared'
1_35.fits: 'Squared'
1_36.fits: 'Squared'
1_37.fits: 'Squared'
1_38.fits: 'Squared'
1_39.fits: 'Squared'
1_40.fits: 'Squared'
1_41.fits: 'Squared'
1_42.fits: 'Squared'
1_43.fits: 'Squared'
1_44.fits: 'Squared'
1_45.fits: 'Squared'
1_46.fits: 'Squared'
1_47.fits: 'Squared'
1_48.fits: 'None'
1_51.fits: 'None'
1_52.fits: 'None'
1_53.fits: 'Squared'
1_54.fits: 'Squared'
1_55.fits: 'Squared'
1_56.fits: 'Squared'
1_57.fits: 'None'

suprimecam/camera
Converting 10 files
Chihiro.fits: 'Squared'
Clarisse.fits: 'Squared'
Fio.fits: 'Squared'
Kiki.fits: 'Squared'
Nausicaa.fits: 'Squared'
Ponyo.fits: 'Squared'
San.fits: 'Squared'
Satsuki.fits: 'Squared'
Sheeta.fits: 'Squared'
Sophie.fits: 'Squared'

suprimecam/mit_camera
Converting 10 files
si001s.fits: 'None'
si002s.fits: 'None'
si005s.fits: 'None'
si006s.fits: 'None'
w4c5.fits: 'None'
w67c1.fits: 'None'
w6c1.fits: 'None'
w7c3.fits: 'None'
w93c2.fits: 'None'
w9c2.fits: 'None'

python/lsst/obs/hsc/hscMapper.py

@@ -4,10 +4,12 @@
 
 import lsst.daf.base as dafBase
 from lsst.daf.butlerUtils import CameraMapper
+from lsst.daf.persistence import ButlerLocation
 import lsst.afw.image.utils as afwImageUtils
 import lsst.afw.image as afwImage
 import lsst.afw.math as afwMath
 import lsst.afw.geom as afwGeom
+from lsst.ip.isr import LinearizeSquared
 import lsst.pex.policy as pexPolicy
 
 class HscMapper(CameraMapper):
@@ -27,6 +29,8 @@ def __init__(self, **kwargs):
 
         super(HscMapper, self).__init__(policy, policyFile.getRepositoryPath(), **kwargs)
 
+        self._linearize = LinearizeSquared()
+
         # Ensure each dataset type of interest knows about the full range of keys available from the registry
         keys = {'field': str,
                 'visit': int,
@@ -255,6 +259,27 @@ def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
         """How many bits are required for the maximum exposure ID"""
         return 32 # just a guess, but this leaves plenty of space for sources
 
+    def map_linearize(self, dataId, write=False):
+        """Map a linearizer."""
+        if self._linearize is None:
+            raise RuntimeError("No linearizer available.")
+        actualId = self._transformId(dataId)
+        return ButlerLocation(
+            pythonType = "lsst.ip.isr.LinearizeSquared",
+            cppType = "Config",
+            storageName = "PickleStorage",
+            locationList = "ignored",
+            dataId = actualId,
+            mapper = self,
+        )
+
+    def bypass_linearize(self, datasetType, pythonType, butlerLocation, dataId):
+        """Return the linearizer.
+        """
+        if self._linearize is None:
+            raise RuntimeError("No linearizer available.")
+        return self._linearize
+
     def _computeCoaddExposureId(self, dataId, singleFilter):
         """Compute the 64-bit (long) identifier for a coadd.
 

python/lsst/obs/subaru/isr.py

@@ -86,6 +86,7 @@ def run(self, exposure):
 class SubaruIsrConfig(IsrTask.ConfigClass):
     qa = pexConfig.ConfigField(doc="QA-related config options", dtype=QaConfig)
     doSaturation = pexConfig.Field(doc="Mask saturated pixels?", dtype=bool, default=True)
+    doSuspect = pexConfig.Field(doc="Mask suspect pixels?", dtype=bool, default=True)
     doWidenSaturationTrails = pexConfig.Field(doc="Widen bleed trails based on their width?",
                                               dtype=bool, default=True)
     doOverscan = pexConfig.Field(doc="Do overscan subtraction?", dtype=bool, default=True)
@@ -102,12 +103,6 @@ class SubaruIsrConfig(IsrTask.ConfigClass):
         doc = "Correct for crosstalk",
         default = True,
     )
-    doLinearize = pexConfig.Field(
-        dtype = bool,
-        doc = "Correct for nonlinearity of the detector's response and set suspect level " \
-                "(ignored if coefficients are 0.0)",
-        default = True,
-    )
     doApplyGains = pexConfig.Field(
         dtype = bool,
         doc = """Correct the amplifiers for their gains
@@ -246,6 +241,8 @@ def runDataRef(self, sensorRef):
             self.measureOverscan(ccdExposure, amp)
             if self.config.doSaturation and not badAmp:
                 self.saturationDetection(ccdExposure, amp)
+            if self.config.doSuspect and not badAmp:
+                self.suspectDetection(ccdExposure, amp)
             if self.config.doOverscan and not badAmp:
                 ampImage = afwImage.MaskedImageF(ccdExposure.getMaskedImage(), amp.getRawDataBBox(),
                                                  afwImage.PARENT)
@@ -285,8 +282,10 @@ def runDataRef(self, sensorRef):
         if self.config.doBias:
             biasExposure = self.getIsrExposure(sensorRef, "bias")
             self.biasCorrection(ccdExposure, biasExposure)
-        if self.config.doLinearize:
-            self.linearize(ccdExposure)
+        if self.doLinearize(ccd):
+            self.log.log(self.log.INFO, "Applying linearity corrections to Ccd %s" % (ccd.getId()))
+            linearizer = sensorRef.get("linearize")
+            linearizer(image=ccdExposure.getMaskedImage().getImage(), detector=ccd, log=self.log)
         if self.config.doCrosstalk:
             self.crosstalk.run(ccdExposure)
 
@@ -527,58 +526,6 @@ def guider(self, exposure):
             "Guider shadow trimming is enabled but no generic implementation is present"
             )
 
-    def linearize(self, exposure):
-        """Correct for non-linearity
-
-        @param exposure Exposure to process
-        """
-        assert exposure, "No exposure provided"
-
-        ccd = exposure.getDetector()
-
-        if ccd.getSerial() in map(str, range(104, 112)):
-            self.log.warn("Skipping linearity correction for focus CCD %s: no coefficients available" %
-                          (ccd.getSerial(),))
-            return
-
-        linearized = False              # did we apply linearity corrections?
-        for amp in ccd:
-            linearityCoefficient = amp.getLinearityCoeffs()[0]
-            linearityThreshold = amp.getLinearityCoeffs()[1]
-            linearityMaxCorrectable = amp.getSuspectLevel()
-            linearityType = amp.getLinearityType()
-
-            ampImage = afwImage.MaskedImageF(exposure.getMaskedImage(), amp.getBBox(),
-                                                 afwImage.PARENT)
-
-            setSuspectPixels = not math.isnan(linearityMaxCorrectable)
-
-            if not setSuspectPixels and linearityCoefficient == 0.0:
-                continue                # nothing to do
-
-            linearized = True
-            #
-            # We may have a max correctable level even if we make no attempt to correct
-            #
-            if setSuspectPixels:
-                afwDetection.FootprintSet(ampImage,
-                                          afwDetection.Threshold(linearityMaxCorrectable), "SUSPECT")
-
-            if linearityType == 'PROPORTIONAL':
-                if linearityThreshold != 0.0:
-                    raise RuntimeError(
-                        ("The threshold for PROPORTIONAL linearity corrections must be 0; saw %g" +
-                         " for ccd %s amp %s") % (linearityThreshold, ccd.getId(), amp.getName()))
-
-                ampArr = ampImage.getImage().getArray()
-                ampArr *= 1.0 + linearityCoefficient*ampArr
-            else:
-                raise NotImplementedError("Unimplemented linearity type: %d", linearityType)
-
-        if linearized:
-            self.log.log(self.log.INFO, "Applying linearity corrections to Ccd %s" % (ccd.getId()))
-
-
     def flatCorrection(self, exposure, flatExposure):
         """Apply flat correction in-place
 

python/lsst/obs/suprimecam/suprimecamMapper.py

@@ -6,7 +6,9 @@
 import lsst.afw.geom as afwGeom
 import lsst.afw.image.utils as afwImageUtils
 
+from lsst.daf.persistence import ButlerLocation
 from lsst.daf.butlerUtils import CameraMapper
+from lsst.ip.isr import LinearizeSquared
 import lsst.pex.policy as pexPolicy
 
 class SuprimecamMapperBase(CameraMapper):
@@ -15,6 +17,8 @@ class SuprimecamMapperBase(CameraMapper):
     def __init__(self, *args, **kwargs):
         super(SuprimecamMapperBase, self).__init__(*args, **kwargs)
 
+        self._linearize = LinearizeSquared()
+
         # Ensure each dataset type of interest knows about the full range of keys available from the registry
         keys = {'field': str,
                 'visit': int,
@@ -217,6 +221,27 @@ def bypass_ccdExposureId(self, datasetType, pythonType, location, dataId):
     def bypass_ccdExposureId_bits(self, datasetType, pythonType, location, dataId):
         return 32 # not really, but this leaves plenty of space for sources
 
+    def map_linearize(self, dataId, write=False):
+        """Map a linearizer."""
+        if self._linearize is None:
+            raise RuntimeError("No linearizer available.")
+        actualId = self._transformId(dataId)
+        return ButlerLocation(
+            pythonType = "lsst.ip.isr.LinearizeSquared",
+            cppType = "Config",
+            storageName = "PickleStorage",
+            locationList = "ignored",
+            dataId = actualId,
+            mapper = self,
+        )
+
+    def bypass_linearize(self, datasetType, pythonType, butlerLocation, dataId):
+        """Return the linearizer.
+        """
+        if self._linearize is None:
+            raise RuntimeError("No linearizer available.")
+        return self._linearize
+
     def _computeStackExposureId(self, dataId):
         """Compute the 64-bit (long) identifier for a Stack exposure.