From d7ef951f2267641795dc447d1687f5011a16dfc9 Mon Sep 17 00:00:00 2001
From: Romain Hugonnet <romain.hugonnet@gmail.com>
Date: Fri, 12 Aug 2022 15:20:01 +0200
Subject: [PATCH] Fix previously skipped tests that used to fail randomly
 (#280)

* fix deramp residual and optimizer

* Unskip test that passes locally to see if passes CI

* Adjust tests with expected behaviour of skimage.warp

* Specify change in test values in comment
---
 tests/test_coreg.py | 22 +++++++++++-----------
 tests/test_fit.py   |  1 -
 2 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/tests/test_coreg.py b/tests/test_coreg.py
index fb32b730..098b6ccf 100644
--- a/tests/test_coreg.py
+++ b/tests/test_coreg.py
@@ -655,16 +655,16 @@ def test_coreg_oneliner(_) -> None:
         assert np.array_equal(dem_arr, dem_arr2_fixed)
 
 
-@pytest.mark.skip('Failure triggered between 10.2021 and 04.2022, likely with warp dem, issue opened.')
 def test_apply_matrix():
     warnings.simplefilter("error")
     ref, tba, outlines = load_examples()  # Load example reference, to-be-aligned and mask.
+    ref_arr = gu.spatial_tools.get_array_and_mask(ref)[0]
 
     # Test only bias (it should just apply the bias and not make anything else)
     bias = 5
     matrix = np.diag(np.ones(4, float))
     matrix[2, 3] = bias
-    transformed_dem = coreg.apply_matrix(ref.data.squeeze(), ref.transform, matrix)
+    transformed_dem = coreg.apply_matrix(ref_arr, ref.transform, matrix)
     reverted_dem = transformed_dem - bias
 
     # Check that the reverted DEM has the exact same values as the initial one
@@ -674,7 +674,7 @@ def test_apply_matrix():
     # Synthesize a shifted and vertically offset DEM
     pixel_shift = 11
     bias = 5
-    shifted_dem = ref.data.squeeze().copy()
+    shifted_dem = ref_arr.copy()
     shifted_dem[:, pixel_shift:] = shifted_dem[:, :-pixel_shift]
     shifted_dem[:, :pixel_shift] = np.nan
     shifted_dem += bias
@@ -683,17 +683,17 @@ def test_apply_matrix():
     matrix[0, 3] = pixel_shift * tba.res[0]
     matrix[2, 3] = -bias
 
-    transformed_dem = coreg.apply_matrix(shifted_dem.data.squeeze(),
+    transformed_dem = coreg.apply_matrix(shifted_dem,
                                          ref.transform, matrix, resampling="bilinear")
 
-    # Dilate the mask a bit to ensure that edge pixels are removed.
+    # Dilate the mask: this should remove the same edge pixels as done by skimage
     transformed_dem_dilated = coreg.apply_matrix(
-        shifted_dem.data.squeeze(),
+        shifted_dem,
         ref.transform, matrix, resampling="bilinear", dilate_mask=True)
-    # Validate that some pixels were removed.
-    assert np.count_nonzero(np.isfinite(transformed_dem)) > np.count_nonzero(np.isfinite(transformed_dem_dilated))
+    # Validate the same pixels were removed.
+    assert np.count_nonzero(np.isfinite(transformed_dem)) == np.count_nonzero(np.isfinite(transformed_dem_dilated))
 
-    diff = np.asarray(ref.data.squeeze() - transformed_dem)
+    diff = np.asarray(ref_arr - transformed_dem)
 
     # Check that the median is very close to zero
     assert np.abs(np.nanmedian(diff)) < 0.01
@@ -779,7 +779,6 @@ def rotation_matrix(rotation=30):
     print(np.nanmedian(diff), spatialstats.nmad(diff))
 
 
-@pytest.mark.skip('Failure triggered between 10.2021 and 04.2022, likely with warp dem, issue opened.')
 def test_warp_dem():
     """Test that the warp_dem function works expectedly."""
     warnings.simplefilter("error")
@@ -822,7 +821,8 @@ def test_warp_dem():
     # The warped DEM should have the value 'elev_shift' in the upper left corner.
     assert warped_dem[0, 0] == elev_shift
     # The corner should be zero, so the corner pixel (represents the corner minus resolution / 2) should be close.
-    assert warped_dem[-1, -1] < 1.0
+    # We select the pixel before the corner (-2 in X-axis) to avoid the NaN propagation on the bottom row.
+    assert warped_dem[-2, -1] < 1
 
     # Synthesise some X/Y/Z coordinates on the DEM.
     source_coords = np.array(
diff --git a/tests/test_fit.py b/tests/test_fit.py
index ce4f47a5..18b46a70 100644
--- a/tests/test_fit.py
+++ b/tests/test_fit.py
@@ -88,7 +88,6 @@ def test_robust_polynomial_fit_noise_and_outliers(self):
         for i in range(3):
             assert coefs6[i+1] == pytest.approx(true_coefs[i+1], abs=1)
 
-    @pytest.mark.skip('This test randomly fails in CI: issue opened.')
     def test_robust_sumsin_fit(self):
 
         # Define X vector