Add DecorrelateALKernelSpatialTask and config

Add unit test for DecorrelateALKernelSpatialTask

Remove redundant logging

python/lsst/ip/diffim/imageDecorrelation.py

@@ -34,10 +34,12 @@
 import lsst.pipe.base as pipeBase
 import lsst.log
 
-from .imageMapReduce import (ImageMapReduceConfig, ImageMapperSubtask)
+from .imageMapReduce import (ImageMapReduceConfig, ImageMapReduceTask,
+                             ImageMapperSubtask)
 
 __all__ = ("DecorrelateALKernelTask", "DecorrelateALKernelConfig",
-           "DecorrelateALKernelMapperSubtask", "DecorrelateALKernelMapReduceConfig")
+           "DecorrelateALKernelMapperSubtask", "DecorrelateALKernelMapReduceConfig",
+           "DecorrelateALKernelSpatialConfig", "DecorrelateALKernelSpatialTask")
 
 
 class DecorrelateALKernelConfig(pexConfig.Config):
@@ -67,7 +69,7 @@ class DecorrelateALKernelTask(pipeBase.Task):
 
     \brief Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
 
-    \section pipe_tasks_multiBand_Contents Contents
+    \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Contents      Contents
 
       - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Purpose
       - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Config
@@ -109,7 +111,6 @@ class DecorrelateALKernelTask(pipeBase.Task):
 
     \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Config       Configuration parameters
 
-    This task currently has no relevant configuration parameters.
     See \ref DecorrelateALKernelConfig
 
     \section ip_diffim_imageDecorrelation_DecorrelateALKernelTask_Debug		Debug variables
@@ -376,9 +377,9 @@ def run(self, subExposure, expandedSubExposure, fullBBox,
 
         Parameters
         ----------
-        subExposure : afw.Exposure
+        subExposure : lsst.afw.image.Exposure
             the sub-exposure of the diffim
-        expandedSubExposure : afw.Exposure
+        expandedSubExposure : lsst.afw.image.Exposure
             the expanded sub-exposure upon which to operate
         fullBBox : afwGeom.BoundingBox
             the bounding box of the original exposure
@@ -393,21 +394,25 @@ def run(self, subExposure, expandedSubExposure, fullBBox,
             `psfMatchingKernel` is not `None`.
         psfMatchingKernel : Alternative parameter for passing the
             A&L `psfMatchingKernel` directly.
+        preConvKernel : If not None, then pre-filtering was applied
+            to science exposure, and this is the pre-convolution
+            kernel.
         kwargs :
             additional keyword arguments propagated from
             `ImageMapReduceTask.run`.
 
         Returns
         -------
-        A `pipeBase.Struct containing the result of the `subExposure`
-        processing, labelled 'subExposure'. It also returns the
-        'decorrelationKernel', although that currently is not used.
+        A `pipeBase.Struct` containing:
+        * `subExposure` : the result of the `subExposure` processing.
+        * `decorrelationKernel` : the decorrelation kernel, currently
+          not used.
 
         Notes
         -----
         This `run` method accepts parameters identical to those of
         `ImageMapperSubtask.run`, since it is called from the
-        `ImageMapperTask`.  See that class for more information.
+        `ImageMapperTask`. See that class for more information.
         """
         templateExposure = template  # input template
         scienceExposure = science  # input science image
@@ -440,3 +445,189 @@ class DecorrelateALKernelMapReduceConfig(ImageMapReduceConfig):
         doc='A&L decorrelation subtask to run on each sub-image',
         target=DecorrelateALKernelMapperSubtask
     )
+
+
+## \addtogroup LSST_task_documentation
+## \{
+## \page DecorrelateALKernelSpatialTask
+## \ref DecorrelateALKernelSpatialTask_ "DecorrelateALKernelSpatialTask"
+##      Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference,
+##      allowing for spatial variations in PSF and noise
+## \}
+
+
+class DecorrelateALKernelSpatialConfig(pexConfig.Config):
+    """Configuration parameters for the DecorrelateALKernelSpatialTask.
+    """
+    decorrelateConfig = pexConfig.ConfigField(
+        dtype=DecorrelateALKernelConfig,
+        doc='DecorrelateALKernel config to use when running on complete exposure (non spatially-varying)',
+    )
+
+    decorrelateMapReduceConfig = pexConfig.ConfigField(
+        dtype=DecorrelateALKernelMapReduceConfig,
+        doc='DecorrelateALKernelMapReduce config to use when running on each sub-image (spatially-varying)',
+    )
+
+    ignoreMaskPlanes = pexConfig.ListField(
+        dtype=str,
+        doc="""Mask planes to ignore for sigma-clipped statistics""",
+        default=("INTRP", "EDGE", "DETECTED", "SAT", "CR", "BAD", "NO_DATA", "DETECTED_NEGATIVE")
+    )
+
+    def setDefaults(self):
+        self.decorrelateMapReduceConfig.gridStepX = self.decorrelateMapReduceConfig.gridStepY = 19
+        self.decorrelateMapReduceConfig.gridSizeX = self.decorrelateMapReduceConfig.gridSizeY = 20
+        self.decorrelateMapReduceConfig.borderSizeX = self.decorrelateMapReduceConfig.borderSizeY = 6
+        self.decorrelateMapReduceConfig.reducerSubtask.reduceOperation = 'average'
+
+
+class DecorrelateALKernelSpatialTask(pipeBase.Task):
+    """!
+    \anchor DecorrelateALKernelSpatialTask_
+
+    \brief Decorrelate the effect of convolution by Alard-Lupton matching kernel in image difference
+
+    \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Contents       Contents
+
+      - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Purpose
+      - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Config
+      - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Run
+      - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Debug
+      - \ref ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Example
+
+    \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Purpose	Description
+
+    Pipe-task that removes the neighboring-pixel covariance in an
+    image difference that are added when the template image is
+    convolved with the Alard-Lupton PSF matching kernel.
+
+    This task is a simple wrapper around \ref
+    ip_diffim_imageDecorrelation_DecorrelateALKernelTask, which takes
+    a `spatiallyVarying` parameter in its `run` method. If it is
+    `False`, then it simply calls the `run` method of \ref
+    DecorrelateALKernelTask. If it is True, then it uses the \ref
+    ip_diffim_imageMapReduce framework to break the exposures into
+    subExposures on a grid, and performs the `run` method of \ref
+    DecorrelateALKernelTask on each subExposure. This enables it to
+    account for spatially-varying PSFs and noise in the exposures when
+    performing the decorrelation.
+
+    \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Initialize       Task initialization
+
+    \copydoc \_\_init\_\_
+
+    \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Run       Invoking the Task
+
+    \copydoc run
+
+    \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Config       Configuration parameters
+
+    See \ref DecorrelateALKernelSpatialConfig
+
+    \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Debug		Debug variables
+
+    This task has no debug variables
+
+    \section ip_diffim_imageDecorrelation_DecorrelateALKernelSpatialTask_Example	Example of using DecorrelateALKernelSpatialTask
+
+    This task has no standalone example, however it is applied as a
+    subtask of \link pipe.tasks.imageDifference.ImageDifferenceTask
+    ImageDifferenceTask\endlink. There is also an example of its use
+    in \ref testImageDecorrelation.
+    """
+    ConfigClass = DecorrelateALKernelSpatialConfig
+    _DefaultName = "ip_diffim_decorrelateALKernelSpatial"
+
+    def __init__(self, *args, **kwargs):
+        """Create the image decorrelation Task
+
+        Parameters
+        ----------
+        args :
+            arguments to be passed to
+            `lsst.pipe.base.task.Task.__init__`
+        kwargs :
+            additional keyword arguments to be passed to
+            `lsst.pipe.base.task.Task.__init__`
+        """
+        pipeBase.Task.__init__(self, *args, **kwargs)
+
+        self.statsControl = afwMath.StatisticsControl()
+        self.statsControl.setNumSigmaClip(3.)
+        self.statsControl.setNumIter(3)
+        self.statsControl.setAndMask(afwImage.MaskU.getPlaneBitMask(self.config.ignoreMaskPlanes))
+
+    def computeVarianceMean(self, exposure):
+        """Compute the mean of the variance plane of `exposure`.
+        """
+        statObj = afwMath.makeStatistics(exposure.getMaskedImage().getVariance(),
+                                         exposure.getMaskedImage().getMask(),
+                                         afwMath.MEANCLIP, self.statsControl)
+        var = statObj.getValue(afwMath.MEANCLIP)
+        return var
+
+    def run(self, scienceExposure, templateExposure, subtractedExposure, psfMatchingKernel,
+            spatiallyVarying=True, doPreConvolve=False):
+        """! Perform decorrelation of an image difference exposure.
+
+        Decorrelates the diffim due to the convolution of the
+        templateExposure with the A&L psfMatchingKernel. If
+        `spatiallyVarying` is True, it utilizes the spatially varying
+        matching kernel via the `imageMapReduce` framework to perform
+        spatially-varying decorrelation on a grid of subExposures.
+
+        Parameters
+        ----------
+        scienceExposure : lsst.afw.image.Exposure
+           the science Exposure used for PSF matching
+        templateExposure : lsst.afw.image.Exposure
+           the template Exposure used for PSF matching
+        subtractedExposure : lsst.afw.image.Exposure
+           the subtracted Exposure produced by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
+        psfMatchingKernel :
+           an (optionally spatially-varying) PSF matching kernel produced
+           by `ip_diffim.ImagePsfMatchTask.subtractExposures()`
+        spatiallyVarying : bool
+           if True, perform the spatially-varying operation
+        doPreConvolve : bool
+           if True, the scienceExposure has been pre-filtered with its PSF. (Currently
+           this option is experimental.)
+
+        Returns
+        -------
+        a `pipeBase.Struct` containing:
+            * `correctedExposure`: the decorrelated diffim
+        """
+        self.log.info('Running A&L decorrelation: spatiallyVarying=%r' % spatiallyVarying)
+
+        svar = self.computeVarianceMean(scienceExposure)
+        tvar = self.computeVarianceMean(templateExposure)
+
+        var = self.computeVarianceMean(subtractedExposure)
+
+        if spatiallyVarying:
+            self.log.info("Variance (science, template): (%f, %f)", svar, tvar)
+            self.log.info("Variance (uncorrected diffim): %f", var)
+            config = self.config.decorrelateMapReduceConfig
+            task = ImageMapReduceTask(config=config)
+            results = task.run(subtractedExposure, science=scienceExposure,
+                               template=templateExposure, psfMatchingKernel=psfMatchingKernel,
+                               preConvKernel=None, forceEvenSized=True)
+            results.correctedExposure = results.exposure
+
+            # Make sure masks of input image are propagated to diffim
+            def gm(exp):
+                return exp.getMaskedImage().getMask()
+            gm(results.correctedExposure)[:, :] = gm(subtractedExposure)
+
+            var = self.computeVarianceMean(results.correctedExposure)
+            self.log.info("Variance (corrected diffim): %f", var)
+
+        else:
+            config = self.config.decorrelateConfig
+            task = DecorrelateALKernelTask(config=config)
+            results = task.run(scienceExposure, templateExposure,
+                               subtractedExposure, psfMatchingKernel)
+
+        return results

tests/testImageDecorrelation.py

@@ -33,7 +33,9 @@
 import lsst.daf.base as dafBase
 
 from lsst.ip.diffim.imageDecorrelation import (DecorrelateALKernelTask,
-                                               DecorrelateALKernelMapReduceConfig)
+                                               DecorrelateALKernelMapReduceConfig,
+                                               DecorrelateALKernelSpatialConfig,
+                                               DecorrelateALKernelSpatialTask)
 from lsst.ip.diffim.imageMapReduce import ImageMapReduceTask
 
 try:
@@ -325,6 +327,7 @@ def _testDecorrelation(self, expected_var, corrected_diffExp):
             print('CORRECTED VARIANCE:', var, mn)
         self.assertFloatsAlmostEqual(mn, expected_var, rtol=0.02)
         self.assertFloatsAlmostEqual(var, mn, rtol=0.05)
+        return var, mn
 
     def _testDiffimCorrection(self, svar, tvar):
         """ Run decorrelation and check the variance of the corrected diffim.
@@ -370,8 +373,8 @@ def _testDiffimCorrection_mapReduced(self, svar, tvar, varyPsf=0.0):
                                            corrected_diffExp_OLD.getMaskedImage())
 
     def testDiffimCorrection_mapReduced(self):
-        """Test decorrelated diffim when using the imageMapReduce task.
-           Compare results with those from the original DecorrelateALKernelTask.
+        """ Test decorrelated diffim when using the imageMapReduce task.
+            Compare results with those from the original DecorrelateALKernelTask.
         """
         # Same variance
         self._testDiffimCorrection_mapReduced(svar=0.04, tvar=0.04)
@@ -380,6 +383,43 @@ def testDiffimCorrection_mapReduced(self):
         # Template variance is higher than that of the science img.
         self._testDiffimCorrection_mapReduced(svar=0.08, tvar=0.04)
 
+    def _runDecorrelationSpatialTask(self, diffExp, mKernel, spatiallyVarying=False):
+        """ Run decorrelation using the DecorrelateALKernelSpatialTask.
+        """
+        config = DecorrelateALKernelSpatialConfig()
+        task = DecorrelateALKernelSpatialTask(config=config)
+        decorrResult = task.run(scienceExposure=self.im1ex, templateExposure=self.im2ex,
+                                subtractedExposure=diffExp, psfMatchingKernel=mKernel,
+                                spatiallyVarying=spatiallyVarying)
+        corrected_diffExp = decorrResult.correctedExposure
+        return corrected_diffExp
+
+    def _testDiffimCorrection_spatialTask(self, svar, tvar, varyPsf=0.0):
+        """Run decorrelation using the DecorrelateALKernelSpatialTask, and
+        check the variance of the corrected diffim. Do it for `spatiallyVarying` both
+        True and False. Also compare the variances between the two `spatiallyVarying`
+        cases.
+        """
+        self._setUpImages(svar=svar, tvar=tvar, varyPsf=varyPsf)
+        diffExp, mKernel, expected_var = self._makeAndTestUncorrectedDiffim()
+        variances = []
+        for spatiallyVarying in [False, True]:
+            corrected_diffExp = self._runDecorrelationSpatialTask(diffExp, mKernel,
+                                                                  spatiallyVarying)
+            var, mn = self._testDecorrelation(expected_var, corrected_diffExp)
+            variances.append(var)
+        self.assertFloatsAlmostEqual(variances[0], variances[1], rtol=0.03)
+
+    def testDiffimCorrection_spatialTask(self):
+        """Test decorrelated diffim when using the DecorrelateALKernelSpatialTask.
+           Compare results with those from the original DecorrelateALKernelTask.
+        """
+        # Same variance
+        self._testDiffimCorrection_spatialTask(svar=0.04, tvar=0.04)
+        # Science image variance is higher than that of the template.
+        self._testDiffimCorrection_spatialTask(svar=0.04, tvar=0.08)
+        # Template variance is higher than that of the science img.
+        self._testDiffimCorrection_spatialTask(svar=0.08, tvar=0.04)
 
 class MemoryTester(lsst.utils.tests.MemoryTestCase):
     pass