diff --git a/doc/source/readers.rst b/doc/source/readers.rst index c1ecd139ec..f96a56e518 100644 --- a/doc/source/readers.rst +++ b/doc/source/readers.rst @@ -63,7 +63,7 @@ polarization:: Or multiple calibrations:: - >>> scn.load([0.6, 10.8], calibrations=['brightness_temperature', 'radiance']) + >>> scn.load([0.6, 10.8], calibration=['brightness_temperature', 'radiance']) In the above case Satpy will load whatever dataset is available and matches the specified parameters. So the above ``load`` call would load the ``0.6`` diff --git a/satpy/resample.py b/satpy/resample.py index 58a6712fdf..348194f12e 100644 --- a/satpy/resample.py +++ b/satpy/resample.py @@ -141,6 +141,8 @@ import dask import dask.array as da import zarr +import pyresample +from packaging import version from pyresample.ewa import fornav, ll2cr from pyresample.geometry import SwathDefinition @@ -177,6 +179,8 @@ resamplers_cache = WeakValueDictionary() +PR_USE_SKIPNA = version.parse(pyresample.__version__) > version.parse("1.17.0") + def hash_dict(the_dict, the_hash=None): """Calculate a hash for a dictionary.""" @@ -1046,6 +1050,26 @@ def compute(self, data, expand=True, **kwargs): return update_resampled_coords(data, new_data, target_geo_def) +def _get_arg_to_pass_for_skipna_handling(**kwargs): + """Determine if skipna can be passed to the compute functions for the average and sum bucket resampler.""" + # FIXME this can be removed once Pyresample 1.18.0 is a Satpy requirement + + if PR_USE_SKIPNA: + if 'mask_all_nan' in kwargs: + warnings.warn('Argument mask_all_nan is deprecated. Please use skipna for missing values handling. ' + 'Continuing with default skipna=True, if not provided differently.', DeprecationWarning) + kwargs.pop('mask_all_nan') + else: + if 'mask_all_nan' in kwargs: + warnings.warn('Argument mask_all_nan is deprecated.' + 'Please update Pyresample and use skipna for missing values handling.', + DeprecationWarning) + kwargs.setdefault('mask_all_nan', False) + kwargs.pop('skipna') + + return kwargs + + class BucketResamplerBase(BaseResampler): """Base class for bucket resampling which implements averaging.""" @@ -1078,6 +1102,11 @@ def resample(self, data, **kwargs): Returns (xarray.DataArray): Data resampled to the target area """ + if not PR_USE_SKIPNA and 'skipna' in kwargs: + raise ValueError('You are trying to set the skipna argument but you are using an old version of' + ' Pyresample that does not support it.' + 'Please update Pyresample to 1.18.0 or higher to be able to use this argument.') + self.precompute(**kwargs) attrs = data.attrs.copy() data_arr = data.data @@ -1128,24 +1157,33 @@ class BucketAvg(BucketResamplerBase): Parameters ---------- fill_value : float (default: np.nan) - Fill value for missing data - mask_all_nans : boolean (default: False) - Mask all locations with all-NaN values + Fill value to mark missing/invalid values in the input data, + as well as in the binned and averaged output data. + skipna : boolean (default: True) + If True, skips missing values (as marked by NaN or `fill_value`) for the average calculation + (similarly to Numpy's `nanmean`). Buckets containing only missing values are set to fill_value. + If False, sets the bucket to fill_value if one or more missing values are present in the bucket + (similarly to Numpy's `mean`). + In both cases, empty buckets are set to `fill_value`. """ - def compute(self, data, fill_value=np.nan, mask_all_nan=False, **kwargs): + def compute(self, data, fill_value=np.nan, skipna=True, **kwargs): """Call the resampling.""" + LOG.debug("Resampling %s", str(data.name)) + + kwargs = _get_arg_to_pass_for_skipna_handling(skipna=skipna, **kwargs) + results = [] if data.ndim == 3: for i in range(data.shape[0]): res = self.resampler.get_average(data[i, :, :], fill_value=fill_value, - mask_all_nan=mask_all_nan) + **kwargs) results.append(res) else: res = self.resampler.get_average(data, fill_value=fill_value, - mask_all_nan=mask_all_nan) + **kwargs) results.append(res) return da.stack(results) @@ -1161,22 +1199,29 @@ class BucketSum(BucketResamplerBase): ---------- fill_value : float (default: np.nan) Fill value for missing data - mask_all_nans : boolean (default: False) - Mask all locations with all-NaN values + skipna : boolean (default: True) + If True, skips NaN values for the sum calculation + (similarly to Numpy's `nansum`). Buckets containing only NaN are set to zero. + If False, sets the bucket to NaN if one or more NaN values are present in the bucket + (similarly to Numpy's `sum`). + In both cases, empty buckets are set to 0. """ - def compute(self, data, mask_all_nan=False, **kwargs): + def compute(self, data, skipna=True, **kwargs): """Call the resampling.""" LOG.debug("Resampling %s", str(data.name)) + + kwargs = _get_arg_to_pass_for_skipna_handling(skipna=skipna, **kwargs) + results = [] if data.ndim == 3: for i in range(data.shape[0]): res = self.resampler.get_sum(data[i, :, :], - mask_all_nan=mask_all_nan) + **kwargs) results.append(res) else: - res = self.resampler.get_sum(data, mask_all_nan=mask_all_nan) + res = self.resampler.get_sum(data, **kwargs) results.append(res) return da.stack(results) diff --git a/satpy/tests/test_resample.py b/satpy/tests/test_resample.py index da0f06f447..cd0fa6d772 100644 --- a/satpy/tests/test_resample.py +++ b/satpy/tests/test_resample.py @@ -726,36 +726,87 @@ def test_precompute(self, bucket): self.assertTrue(self.bucket.resampler) bucket.assert_called_once_with(self.target_geo_def, 1, 2) + def _compute_mocked_bucket_avg(self, data, return_data=None, **kwargs): + """Compute the mocked bucket average.""" + self.bucket.resampler = mock.MagicMock() + if return_data is not None: + self.bucket.resampler.get_average.return_value = return_data + else: + self.bucket.resampler.get_average.return_value = data + res = self.bucket.compute(data, **kwargs) + return res + def test_compute(self): """Test bucket resampler computation.""" import dask.array as da # 1D data - self.bucket.resampler = mock.MagicMock() data = da.ones((5,)) - self.bucket.resampler.get_average.return_value = data - res = self.bucket.compute(data, fill_value=2) - self.bucket.resampler.get_average.assert_called_once_with( - data, - fill_value=2, - mask_all_nan=False) + res = self._compute_mocked_bucket_avg(data, fill_value=2) self.assertEqual(res.shape, (1, 5)) # 2D data - self.bucket.resampler = mock.MagicMock() data = da.ones((5, 5)) - self.bucket.resampler.get_average.return_value = data - res = self.bucket.compute(data, fill_value=2) - self.bucket.resampler.get_average.assert_called_once_with( - data, - fill_value=2, - mask_all_nan=False) + res = self._compute_mocked_bucket_avg(data, fill_value=2) self.assertEqual(res.shape, (1, 5, 5)) # 3D data - self.bucket.resampler = mock.MagicMock() data = da.ones((3, 5, 5)) self.bucket.resampler.get_average.return_value = data[0, :, :] - res = self.bucket.compute(data, fill_value=2) + res = self._compute_mocked_bucket_avg(data, return_data=data[0, :, :], fill_value=2) self.assertEqual(res.shape, (3, 5, 5)) + @mock.patch('satpy.resample.PR_USE_SKIPNA', True) + def test_compute_and_use_skipna_handling(self): + """Test bucket resampler computation and use skipna handling.""" + import dask.array as da + data = da.ones((5,)) + + self._compute_mocked_bucket_avg(data, fill_value=2, mask_all_nan=True) + self.bucket.resampler.get_average.assert_called_once_with( + data, + fill_value=2, + skipna=True) + + self._compute_mocked_bucket_avg(data, fill_value=2, skipna=False) + self.bucket.resampler.get_average.assert_called_once_with( + data, + fill_value=2, + skipna=False) + + self._compute_mocked_bucket_avg(data, fill_value=2) + self.bucket.resampler.get_average.assert_called_once_with( + data, + fill_value=2, + skipna=True) + + @mock.patch('satpy.resample.PR_USE_SKIPNA', False) + def test_compute_and_not_use_skipna_handling(self): + """Test bucket resampler computation and not use skipna handling.""" + import dask.array as da + data = da.ones((5,)) + + self._compute_mocked_bucket_avg(data, fill_value=2, mask_all_nan=True) + self.bucket.resampler.get_average.assert_called_once_with( + data, + fill_value=2, + mask_all_nan=True) + + self._compute_mocked_bucket_avg(data, fill_value=2, mask_all_nan=False) + self.bucket.resampler.get_average.assert_called_once_with( + data, + fill_value=2, + mask_all_nan=False) + + self._compute_mocked_bucket_avg(data, fill_value=2) + self.bucket.resampler.get_average.assert_called_once_with( + data, + fill_value=2, + mask_all_nan=False) + + self._compute_mocked_bucket_avg(data, fill_value=2, skipna=True) + self.bucket.resampler.get_average.assert_called_once_with( + data, + fill_value=2, + mask_all_nan=False) + @mock.patch('pyresample.bucket.BucketResampler') def test_resample(self, pyresample_bucket): """Test bucket resamplers resample method.""" @@ -812,34 +863,80 @@ def setUp(self): self.target_geo_def = mock.MagicMock(get_lonlats=get_lonlats) self.bucket = BucketSum(self.source_geo_def, self.target_geo_def) + def _compute_mocked_bucket_sum(self, data, return_data=None, **kwargs): + """Compute the mocked bucket sum.""" + self.bucket.resampler = mock.MagicMock() + if return_data is not None: + self.bucket.resampler.get_sum.return_value = return_data + else: + self.bucket.resampler.get_sum.return_value = data + res = self.bucket.compute(data, **kwargs) + return res + def test_compute(self): """Test sum bucket resampler computation.""" import dask.array as da # 1D data - self.bucket.resampler = mock.MagicMock() data = da.ones((5,)) - self.bucket.resampler.get_sum.return_value = data - res = self.bucket.compute(data) - self.bucket.resampler.get_sum.assert_called_once_with( - data, - mask_all_nan=False) + res = self._compute_mocked_bucket_sum(data) self.assertEqual(res.shape, (1, 5)) # 2D data - self.bucket.resampler = mock.MagicMock() data = da.ones((5, 5)) - self.bucket.resampler.get_sum.return_value = data - res = self.bucket.compute(data) - self.bucket.resampler.get_sum.assert_called_once_with( - data, - mask_all_nan=False) + res = self._compute_mocked_bucket_sum(data) self.assertEqual(res.shape, (1, 5, 5)) # 3D data - self.bucket.resampler = mock.MagicMock() data = da.ones((3, 5, 5)) - self.bucket.resampler.get_sum.return_value = data[0, :, :] - res = self.bucket.compute(data) + res = self._compute_mocked_bucket_sum(data, return_data=data[0, :, :]) self.assertEqual(res.shape, (3, 5, 5)) + @mock.patch('satpy.resample.PR_USE_SKIPNA', True) + def test_compute_and_use_skipna_handling(self): + """Test bucket resampler computation and use skipna handling.""" + import dask.array as da + data = da.ones((5,)) + + self._compute_mocked_bucket_sum(data, mask_all_nan=True) + self.bucket.resampler.get_sum.assert_called_once_with( + data, + skipna=True) + + self._compute_mocked_bucket_sum(data, skipna=False) + self.bucket.resampler.get_sum.assert_called_once_with( + data, + skipna=False) + + self._compute_mocked_bucket_sum(data) + self.bucket.resampler.get_sum.assert_called_once_with( + data, + skipna=True) + + @mock.patch('satpy.resample.PR_USE_SKIPNA', False) + def test_compute_and_not_use_skipna_handling(self): + """Test bucket resampler computation and not use skipna handling.""" + import dask.array as da + data = da.ones((5,)) + + self._compute_mocked_bucket_sum(data, mask_all_nan=True) + self.bucket.resampler.get_sum.assert_called_once_with( + data, + mask_all_nan=True) + + self._compute_mocked_bucket_sum(data, mask_all_nan=False) + self.bucket.resampler.get_sum.assert_called_once_with( + data, + mask_all_nan=False) + + self._compute_mocked_bucket_sum(data) + self.bucket.resampler.get_sum.assert_called_once_with( + data, + mask_all_nan=False) + + self._compute_mocked_bucket_sum(data, fill_value=2, skipna=True) + self.bucket.resampler.get_sum.assert_called_once_with( + data, + fill_value=2, + mask_all_nan=False) + class TestBucketCount(unittest.TestCase): """Test the count bucket resampler.""" @@ -853,28 +950,32 @@ def setUp(self): self.target_geo_def = mock.MagicMock(get_lonlats=get_lonlats) self.bucket = BucketCount(self.source_geo_def, self.target_geo_def) + def _compute_mocked_bucket_count(self, data, return_data=None, **kwargs): + """Compute the mocked bucket count.""" + self.bucket.resampler = mock.MagicMock() + if return_data is not None: + self.bucket.resampler.get_count.return_value = return_data + else: + self.bucket.resampler.get_count.return_value = data + res = self.bucket.compute(data, **kwargs) + return res + def test_compute(self): """Test count bucket resampler computation.""" import dask.array as da # 1D data - self.bucket.resampler = mock.MagicMock() data = da.ones((5,)) - self.bucket.resampler.get_count.return_value = data - res = self.bucket.compute(data) + res = self._compute_mocked_bucket_count(data) self.bucket.resampler.get_count.assert_called_once_with() self.assertEqual(res.shape, (1, 5)) # 2D data - self.bucket.resampler = mock.MagicMock() data = da.ones((5, 5)) - self.bucket.resampler.get_count.return_value = data - res = self.bucket.compute(data) + res = self._compute_mocked_bucket_count(data) self.bucket.resampler.get_count.assert_called_once_with() self.assertEqual(res.shape, (1, 5, 5)) # 3D data - self.bucket.resampler = mock.MagicMock() data = da.ones((3, 5, 5)) - self.bucket.resampler.get_count.return_value = data[0, :, :] - res = self.bucket.compute(data) + res = self._compute_mocked_bucket_count(data, return_data=data[0, :, :]) self.assertEqual(res.shape, (3, 5, 5))