Fixes and improvement of mask handling in lazy decomposition

CHANGES.rst

@@ -26,6 +26,7 @@ RELEASE_next_patch (Unreleased)
 * Improve error message when initialising SpanROI with left >= right (`#2604 <https://github.com/hyperspy/hyperspy/pull/2604>`_)
 * Fix various bugs with `CircleWidget` and `Line2DWidget` (`#2625 <https://github.com/hyperspy/hyperspy/pull/2625>`_)
 * Allow running the test suite without the pytest-mpl plugin (`#2624 <https://github.com/hyperspy/hyperspy/pull/2624>`_)
+* Fix and improve mask handling in lazy decomposition; Close `#2605 <https://github.com/hyperspy/hyperspy/issues/2605>`_ (`#2657 <https://github.com/hyperspy/hyperspy/pull/2657>`_)
 * Plot scalebar when the axis scale have different sign, fixes `#2557 <https://github.com/hyperspy/hyperspy/issues/2557>`_ (#2657 `<https://github.com/hyperspy/hyperspy/pull/2657>`_)
 
 Changelog

hyperspy/_signals/lazy.py

@@ -736,10 +736,10 @@ def decomposition(
             increased to contain at least ``output_dimension`` signals.
         navigation_mask : {BaseSignal, numpy array, dask array}
             The navigation locations marked as True are not used in the
-            decomposition.
+            decomposition. Not implemented for the 'SVD' algorithm.
         signal_mask : {BaseSignal, numpy array, dask array}
             The signal locations marked as True are not used in the
-            decomposition.
+            decomposition. Not implemented for the 'SVD' algorithm.
         reproject : bool, default True
             Reproject data on the learnt components (factors) after learning.
         print_info : bool, default True
@@ -808,9 +808,9 @@ def decomposition(
         # Initialize print_info
         to_print = [
             "Decomposition info:",
-            "  normalize_poissonian_noise={}".format(normalize_poissonian_noise),
-            "  algorithm={}".format(algorithm),
-            "  output_dimension={}".format(output_dimension)
+            f"  normalize_poissonian_noise={normalize_poissonian_noise}",
+            f"  algorithm={algorithm}",
+            f"  output_dimension={output_dimension}"
         ]
 
         # LEARN
@@ -886,7 +886,7 @@ def decomposition(
                 try:
                     self._unfolded4decomposition = self.unfold()
                     # TODO: implement masking
-                    if navigation_mask or signal_mask:
+                    if navigation_mask is not None or signal_mask is not None:
                         raise NotImplementedError("Masking is not yet implemented for lazy SVD")
 
                     U, S, V = svd(self.data)
@@ -908,6 +908,8 @@ def decomposition(
                         self.fold()
                         self._unfolded4decomposition is False
             else:
+                self._check_navigation_mask(navigation_mask)
+                self._check_signal_mask(signal_mask)
                 this_data = []
                 try:
                     for chunk in progressbar(

hyperspy/signal.py

@@ -6018,16 +6018,17 @@ def _check_navigation_mask(self, mask):
         """
         if isinstance(mask, BaseSignal):
             if mask.axes_manager.signal_dimension != 0:
-                raise ValueError("`mask` must be a signal with "
-                                 "`signal_dimension` equal to 0")
+                raise ValueError("The navigation mask signal must have the "
+                                 "`signal_dimension` equal to 0.")
             elif (mask.axes_manager.navigation_shape !=
                   self.axes_manager.navigation_shape):
-                raise ValueError("`mask` must be a signal with the same "
-                                 "`navigation_shape` as the current signal.")
+                raise ValueError("The navigation mask signal must have the "
+                                 "same `navigation_shape` as the current "
+                                 "signal.")
         if isinstance(mask, np.ndarray) and (
                 mask.shape != self.axes_manager.navigation_shape):
-            raise ValueError("The shape of `mask` must match the shape of "
-                             "the `navigation_shape`.")
+            raise ValueError("The shape of the navigation mask array must "
+                             "match `navigation_shape`.")
 
     def _check_signal_mask(self, mask):
         """
@@ -6050,16 +6051,16 @@ def _check_signal_mask(self, mask):
         """
         if isinstance(mask, BaseSignal):
             if mask.axes_manager.navigation_dimension != 0:
-                raise ValueError("`mask` must be a signal with "
-                                 "`navigation_dimension` equal to 0")
+                raise ValueError("The signal mask signal must have the "
+                                 "`navigation_dimension` equal to 0.")
             elif (mask.axes_manager.signal_shape !=
                   self.axes_manager.signal_shape):
-                raise ValueError("`mask` must be a signal with the same "
+                raise ValueError("The signal mask signal must have the same "
                                  "`signal_shape` as the current signal.")
         if isinstance(mask, np.ndarray) and (
                 mask.shape != self.axes_manager.signal_shape):
-            raise ValueError("The shape of `mask` must match the shape of "
-                             "the `signal_shape`.")
+            raise ValueError("The shape of signal mask array must match "
+                             "`signal_shape`.")
 
 
 ARITHMETIC_OPERATORS = (

hyperspy/tests/learn/test_lazy_decomposition.py

@@ -105,6 +105,42 @@ def test_pca(self, normalize_poissonian_noise):
             explained_variance_norm[: self.rank].sum(), 1.0, atol=1e-6
         )
 
+    @pytest.mark.skipif(not sklearn_installed, reason="sklearn not installed")
+    def test_pca_mask(self):
+        s = self.s
+        sig_mask = (s.inav[0, 0].data < 1.0).compute()
+
+        s.decomposition(output_dimension=3,
+                        algorithm="PCA",
+                        signal_mask=sig_mask)
+        factors = s.learning_results.factors
+        loadings = s.learning_results.loadings
+        _ = loadings @ factors.T
+
+        # Check singular values
+        explained_variance = s.learning_results.explained_variance
+        explained_variance_norm = explained_variance / np.sum(explained_variance)
+        np.testing.assert_allclose(
+            explained_variance_norm[: self.rank].sum(), 1.0, atol=1e-6
+        )
+
+        nav_mask = (s.isig[0].data < 1.0).compute()
+
+        s.decomposition(output_dimension=3,
+                        algorithm="PCA",
+                        navigation_mask=nav_mask)
+        factors = s.learning_results.factors
+        loadings = s.learning_results.loadings
+        _ = loadings @ factors.T
+
+        # Check singular values
+        explained_variance = s.learning_results.explained_variance
+        explained_variance_norm = explained_variance / np.sum(explained_variance)
+        np.testing.assert_allclose(
+            explained_variance_norm[: self.rank].sum(), 1.0, atol=1e-6
+        )
+
+
     @pytest.mark.parametrize("normalize_poissonian_noise", [True, False])
     def test_orpca(self, normalize_poissonian_noise):
         self.s.decomposition(
@@ -203,3 +239,24 @@ def test_no_print(self, algorithm, capfd):
         self.s.decomposition(algorithm=algorithm, output_dimension=2, print_info=False)
         captured = capfd.readouterr()
         assert "Decomposition info:" not in captured.out
+
+    def test_decomposition_mask_SVD(self):
+        s = self.s
+        sig_mask = (s.inav[0].data < 0.5).compute()
+        with pytest.raises(NotImplementedError):
+            s.decomposition(algorithm="SVD", signal_mask=sig_mask)
+
+        nav_mask = (s.isig[0].data < 0.5).compute()
+        with pytest.raises(NotImplementedError):
+            s.decomposition(algorithm="SVD", navigation_mask=nav_mask)
+
+    @pytest.mark.skipif(not sklearn_installed, reason="sklearn not installed")
+    def test_decomposition_mask_wrong_Shape(self):
+        s = self.s
+        sig_mask = (s.inav[0].data < 0.5).compute()[:-2]
+        with pytest.raises(ValueError):
+            s.decomposition(algorithm='PCA', signal_mask=sig_mask)
+
+        nav_mask = (s.isig[0].data < 0.5).compute()[:-2]
+        with pytest.raises(ValueError):
+            s.decomposition(algorithm='PCA', navigation_mask=nav_mask)

hyperspy/tests/signal/test_check_mask.py

@@ -30,18 +30,18 @@ def test_check_navigation_mask():
     assert s._check_navigation_mask(mask) is None
 
     # wrong navigation shape
-    error_message = '`mask` must be a signal with the same `navigation_shape`'
+    error_message = 'The navigation mask signal must have the same'
     with pytest.raises(ValueError, match=error_message):
         s._check_navigation_mask(mask.inav[:-2, :])
 
     # wrong array shape
-    error_message = 'The shape of `mask` must match the shape of the `navigation_shape`.'
+    error_message = 'The shape of the navigation mask array must match'
     with pytest.raises(ValueError, match=error_message):
         s._check_navigation_mask(mask.inav[:-2, :].data)
 
     # wrong signal dimenstion
     mask = (s > 1)
-    error_message = '`mask` must be a signal with `signal_dimension`'
+    error_message = 'The navigation mask signal must have the `signal_dimension`'
     with pytest.raises(ValueError, match=error_message):
         s._check_navigation_mask(mask)
 
@@ -54,18 +54,18 @@ def test_check_signal_mask():
     mask = (s.inav[0, 0] > 1)
     assert s._check_signal_mask(mask) is None
 
-    # wrong navigation shape
-    error_message = '`mask` must be a signal with the same `signal_shape`'
+    # wrong signal shape
+    error_message = 'The signal mask signal must have the same'
     with pytest.raises(ValueError, match=error_message):
         s._check_signal_mask(mask.isig[:-2])
 
     # wrong array shape
-    error_message = 'The shape of `mask` must match the shape of the `signal_shape`.'
+    error_message = 'The shape of signal mask array must match '
     with pytest.raises(ValueError, match=error_message):
         s._check_signal_mask(mask.isig[:-2].data)
 
-    # wrong signal dimenstion
+    # wrong navigation dimenstion
     mask = (s > 1)
-    error_message = '`mask` must be a signal with `navigation_dimension`'
+    error_message = 'The signal mask signal must have the `navigation_dimension`'
     with pytest.raises(ValueError, match=error_message):
         s._check_signal_mask(mask)