From 1ec54cf75a122d9a51b6e7874f0557f6f3635152 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Sat, 12 Nov 2022 19:43:33 +0100 Subject: [PATCH 01/30] Extend blend method and add a new blend function using weights Signed-off-by: Adam.Dybbroe --- satpy/multiscene.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/satpy/multiscene.py b/satpy/multiscene.py index 2dee0ce146..397fc79e93 100644 --- a/satpy/multiscene.py +++ b/satpy/multiscene.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2016-2019 Satpy developers +# Copyright (c) 2016-2019, 2022 Satpy developers # # This file is part of satpy. # @@ -45,6 +45,14 @@ log = logging.getLogger(__name__) +def weighted(datasets, weights=None): + """Blend datasets using weights.""" + indices = np.argmax(np.dstack(weights), axis=-1) + + base = np.choose(indices, datasets) + return base + + def stack(datasets): """Overlay series of datasets on top of each other.""" base = datasets[0].copy() @@ -339,7 +347,7 @@ def resample(self, destination=None, **kwargs): """Resample the multiscene.""" return self._generate_scene_func(self._scenes, 'resample', True, destination=destination, **kwargs) - def blend(self, blend_function=stack): + def blend(self, blend_function=stack, **kwargs): """Blend the datasets into one scene. Reduce the :class:`MultiScene` to a single :class:`~satpy.scene.Scene`. Datasets @@ -364,7 +372,7 @@ def blend(self, blend_function=stack): common_datasets = self.shared_dataset_ids for ds_id in common_datasets: datasets = [scn[ds_id] for scn in self.scenes if ds_id in scn] - new_scn[ds_id] = blend_function(datasets) + new_scn[ds_id] = blend_function(datasets, **kwargs) return new_scn @@ -508,7 +516,7 @@ def _get_single_frame(self, ds, enh_args, fill_value): enh_args = enh_args.copy() # don't change caller's dict! if "decorate" in enh_args: enh_args["decorate"] = self._format_decoration( - ds, enh_args["decorate"]) + ds, enh_args["decorate"]) img = get_enhanced_image(ds, **enh_args) data, mode = img.finalize(fill_value=fill_value) if data.ndim == 3: @@ -632,7 +640,7 @@ def _get_writers_and_frames( info_datasets = [scn.get(dataset_id) for scn in info_scenes] this_fn, shape, this_fill = self._get_animation_info(info_datasets, filename, fill_value=fill_value) data_to_write = self._get_animation_frames( - all_datasets, shape, this_fill, ignore_missing, enh_args) + all_datasets, shape, this_fill, ignore_missing, enh_args) writer = imageio.get_writer(this_fn, **imio_args) frames[dataset_id] = data_to_write @@ -703,8 +711,8 @@ def save_animation(self, filename, datasets=None, fps=10, fill_value=None, raise ImportError("Missing required 'imageio' library") (writers, frames) = self._get_writers_and_frames( - filename, datasets, fill_value, ignore_missing, - enh_args, imio_args={"fps": fps, **kwargs}) + filename, datasets, fill_value, ignore_missing, + enh_args, imio_args={"fps": fps, **kwargs}) client = self._get_client(client=client) # get an ordered list of frames From 7eba546305151bf46caa27f71dec41b5168ba5ec Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Mon, 14 Nov 2022 11:48:19 +0100 Subject: [PATCH 02/30] refactored new function weighted function and accompanying unittests a bit Signed-off-by: Adam.Dybbroe --- satpy/multiscene.py | 10 ++- satpy/tests/test_multiscene.py | 136 ++++++++++++++++++++++++++------- 2 files changed, 116 insertions(+), 30 deletions(-) diff --git a/satpy/multiscene.py b/satpy/multiscene.py index 397fc79e93..ba96570bbe 100644 --- a/satpy/multiscene.py +++ b/satpy/multiscene.py @@ -47,10 +47,12 @@ def weighted(datasets, weights=None): """Blend datasets using weights.""" - indices = np.argmax(np.dstack(weights), axis=-1) - - base = np.choose(indices, datasets) - return base + indices = da.argmax(da.dstack(weights), axis=-1) + dims = datasets[0].dims + attrs = datasets[0].attrs + weighted_array = xr.DataArray(da.choose(indices, datasets), + dims=dims, attrs=attrs) + return weighted_array def stack(datasets): diff --git a/satpy/tests/test_multiscene.py b/satpy/tests/test_multiscene.py index 1d14d43298..6b67bea219 100644 --- a/satpy/tests/test_multiscene.py +++ b/satpy/tests/test_multiscene.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2018 Satpy developers +# Copyright (c) 2018, 2022 Satpy developers # # This file is part of satpy. # @@ -24,8 +24,10 @@ from datetime import datetime from unittest import mock +import dask.array as da import pytest import xarray as xr +from pyresample.geometry import AreaDefinition from satpy import DataQuery from satpy.dataset.dataid import DataID, ModifierTuple, WavelengthRange @@ -166,7 +168,7 @@ def test_from_files(self): "OR_ABI-L1b-RadC-M3C01_G16_s20171171517203_e20171171519577_c20171171520019.nc", "OR_ABI-L1b-RadC-M3C01_G16_s20171171522203_e20171171524576_c20171171525020.nc", "OR_ABI-L1b-RadC-M3C01_G16_s20171171527203_e20171171529576_c20171171530017.nc", - ] + ] input_files_glm = [ "OR_GLM-L2-GLMC-M3_G16_s20171171500000_e20171171501000_c20380190314080.nc", "OR_GLM-L2-GLMC-M3_G16_s20171171501000_e20171171502000_c20380190314080.nc", @@ -179,9 +181,9 @@ def test_from_files(self): ] with mock.patch('satpy.multiscene.Scene') as scn_mock: mscn = MultiScene.from_files( - input_files_abi, - reader='abi_l1b', - scene_kwargs={"reader_kwargs": {}}) + input_files_abi, + reader='abi_l1b', + scene_kwargs={"reader_kwargs": {}}) assert len(mscn.scenes) == 6 calls = [mock.call( filenames={'abi_l1b': [in_file_abi]}, @@ -192,11 +194,11 @@ def test_from_files(self): scn_mock.reset_mock() with pytest.warns(DeprecationWarning): mscn = MultiScene.from_files( - input_files_abi + input_files_glm, - reader=('abi_l1b', "glm_l2"), - group_keys=["start_time"], - ensure_all_readers=True, - time_threshold=30) + input_files_abi + input_files_glm, + reader=('abi_l1b', "glm_l2"), + group_keys=["start_time"], + ensure_all_readers=True, + time_threshold=30) assert len(mscn.scenes) == 2 calls = [mock.call( filenames={'abi_l1b': [in_file_abi], 'glm_l2': [in_file_glm]}) @@ -206,11 +208,11 @@ def test_from_files(self): scn_mock.assert_has_calls(calls) scn_mock.reset_mock() mscn = MultiScene.from_files( - input_files_abi + input_files_glm, - reader=('abi_l1b', "glm_l2"), - group_keys=["start_time"], - ensure_all_readers=False, - time_threshold=30) + input_files_abi + input_files_glm, + reader=('abi_l1b', "glm_l2"), + group_keys=["start_time"], + ensure_all_readers=False, + time_threshold=30) assert len(mscn.scenes) == 12 @@ -531,7 +533,7 @@ def test_crop(self): x_size // 2, y_size // 2, area_extent, - ) + ) scene1["1"] = DataArray(np.zeros((y_size, x_size))) scene1["2"] = DataArray(np.zeros((y_size, x_size)), dims=('y', 'x')) scene1["3"] = DataArray(np.zeros((y_size, x_size)), dims=('y', 'x'), @@ -562,19 +564,29 @@ def setUp(self): import dask.array as da import xarray as xr from pyresample.geometry import AreaDefinition + shape = (8, 12) area = AreaDefinition('test', 'test', 'test', {'proj': 'geos', 'lon_0': -95.5, 'h': 35786023.0}, - 2, 2, [-200, -200, 200, 200]) - ds1 = xr.DataArray(da.zeros((2, 2), chunks=-1), dims=('y', 'x'), + shape[1], shape[0], [-200, -200, 200, 200]) + ds1 = xr.DataArray(da.ones(shape, chunks=-1), dims=('y', 'x'), attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) self.ds1 = ds1 - ds2 = xr.DataArray(da.zeros((2, 2), chunks=-1), dims=('y', 'x'), + wgt1 = xr.DataArray(da.ones(shape, chunks=-1), dims=('y', 'x'), + attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) + self.ds1_wgt = wgt1 + ds2 = xr.DataArray(da.ones(shape, chunks=-1) * 2, dims=('y', 'x'), attrs={'start_time': datetime(2018, 1, 1, 1, 0, 0), 'area': area}) self.ds2 = ds2 - ds3 = xr.DataArray(da.zeros((2, 2), chunks=-1), dims=('y', 'time'), + wgt2 = xr.DataArray(da.zeros(shape, chunks=-1), dims=('y', 'x'), + attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) + self.line = 2 + wgt2[self.line, :] = 2 + self.ds2_wgt = wgt2 + + ds3 = xr.DataArray(da.zeros(shape, chunks=-1), dims=('y', 'time'), attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) self.ds3 = ds3 - ds4 = xr.DataArray(da.zeros((2, 2), chunks=-1), dims=('y', 'time'), + ds4 = xr.DataArray(da.zeros(shape, chunks=-1), dims=('y', 'time'), attrs={'start_time': datetime(2018, 1, 1, 1, 0, 0), 'area': area}) self.ds4 = ds4 @@ -597,6 +609,78 @@ def test_timeseries(self): self.assertTupleEqual((self.ds3.shape[0], self.ds3.shape[1]+self.ds4.shape[1]), res2.shape) +@pytest.fixture +def datasets_and_weights(): + """X-Array datasets with area definition plus weights for input to tests.""" + shape = (8, 12) + area = AreaDefinition('test', 'test', 'test', + {'proj': 'geos', 'lon_0': -95.5, 'h': 35786023.0}, + shape[1], shape[0], [-200, -200, 200, 200]) + ds1 = xr.DataArray(da.ones(shape, chunks=-1), dims=('y', 'x'), + attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) + ds2 = xr.DataArray(da.ones(shape, chunks=-1) * 2, dims=('y', 'x'), + attrs={'start_time': datetime(2018, 1, 1, 1, 0, 0), 'area': area}) + ds3 = xr.DataArray(da.ones(shape, chunks=-1) * 3, dims=('y', 'x'), + attrs={'start_time': datetime(2018, 1, 1, 1, 0, 0), 'area': area}) + + wgt1 = xr.DataArray(da.ones(shape, chunks=-1), dims=('y', 'x'), + attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) + wgt2 = xr.DataArray(da.zeros(shape, chunks=-1), dims=('y', 'x'), + attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) + wgt3 = xr.DataArray(da.zeros(shape, chunks=-1), dims=('y', 'x'), + attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) + + datastruct = {'shape': shape, + 'area': area, + 'datasets': [ds1, ds2, ds3], + 'weights': [wgt1, wgt2, wgt3]} + return datastruct + + +@pytest.mark.parametrize(('line', 'column',), + [(2, 3), (4, 5)] + ) +def test_blend_function_weighted(datasets_and_weights, line, column): + """Test the 'weighted' function.""" + from satpy.multiscene import weighted + + input_data = datasets_and_weights + + input_data['weights'][1][line, :] = 2 + input_data['weights'][2][:, column] = 2 + + blend_result = weighted(input_data['datasets'], input_data['weights']) + + ds1 = input_data['datasets'][0] + ds2 = input_data['datasets'][1] + ds3 = input_data['datasets'][2] + expected = ds1.copy() + expected[:, column] = ds3[:, column] + expected[line, :] = ds2[line, :] + + xr.testing.assert_equal(blend_result.compute(), expected.compute()) + assert expected.attrs == blend_result.attrs + + +def test_blend_function_stack(datasets_and_weights): + """Test the 'stack' function.""" + from satpy.multiscene import stack + + input_data = datasets_and_weights + + ds1 = input_data['datasets'][0] + ds2 = input_data['datasets'][1] + + res = stack([ds1, ds2]) + expected = ds2.copy() + + xr.testing.assert_equal(res.compute(), expected.compute()) + # assert expected.attrs == res.attrs + # FIXME! Looks like the attributes are taken from the first dataset. Should + # be like that? So in this case the datetime is different from "expected" + # (= in this case the last dataset in the stack, the one on top) + + @mock.patch('satpy.multiscene.get_enhanced_image') def test_save_mp4(smg, tmp_path): """Save a series of fake scenes to an mp4 video.""" @@ -656,11 +740,11 @@ def test_save_mp4(smg, tmp_path): with mock.patch('satpy.multiscene.imageio.get_writer') as get_writer: get_writer.return_value = writer_mock mscn.save_animation( - fn, client=False, - enh_args={"decorate": { - "decorate": [{ - "text": { - "txt": + fn, client=False, + enh_args={"decorate": { + "decorate": [{ + "text": { + "txt": "Test {start_time:%Y-%m-%d %H:%M} - " "{end_time:%Y-%m-%d %H:%M}"}}]}}) assert writer_mock.append_data.call_count == 2 + 2 From a7d5ee757ae9acbfbbc7388b96f9f72ec86d891b Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Wed, 16 Nov 2022 08:30:57 +0100 Subject: [PATCH 03/30] Fixing the stacked blend functions taking care of fillevalue and improve tests Signed-off-by: Adam.Dybbroe --- satpy/multiscene.py | 16 +- satpy/tests/test_multiscene.py | 319 ++++++++++++++++++++++----------- 2 files changed, 226 insertions(+), 109 deletions(-) diff --git a/satpy/multiscene.py b/satpy/multiscene.py index ba96570bbe..89b232380d 100644 --- a/satpy/multiscene.py +++ b/satpy/multiscene.py @@ -45,8 +45,15 @@ log = logging.getLogger(__name__) -def weighted(datasets, weights=None): - """Blend datasets using weights.""" +def stack_weighted(datasets, weights=None): + """Stack datasets using weights.""" + # Go through weights and set to zero where corresponding datasets have a value equals _FillValue or nan + for i, dataset in enumerate(datasets): + try: + weights[i] = xr.where(dataset == dataset._FillValue, 0, weights[i]) + except AttributeError: + weights[i] = xr.where(dataset.isnull(), 0, weights[i]) + indices = da.argmax(da.dstack(weights), axis=-1) dims = datasets[0].dims attrs = datasets[0].attrs @@ -59,7 +66,10 @@ def stack(datasets): """Overlay series of datasets on top of each other.""" base = datasets[0].copy() for dataset in datasets[1:]: - base = base.where(dataset.isnull(), dataset) + try: + base = base.where(dataset == dataset._FillValue, dataset) + except AttributeError: + base = base.where(dataset.isnull(), dataset) return base diff --git a/satpy/tests/test_multiscene.py b/satpy/tests/test_multiscene.py index 6b67bea219..deb7cd4bc6 100644 --- a/satpy/tests/test_multiscene.py +++ b/satpy/tests/test_multiscene.py @@ -90,12 +90,30 @@ def _create_test_area(proj_str=None, shape=DEFAULT_SHAPE, extents=None): ) -def _create_test_dataset(name, shape=DEFAULT_SHAPE, area=None): +def _create_test_int8_dataset(name, shape=DEFAULT_SHAPE, area=None, values=None): """Create a test DataArray object.""" import dask.array as da import numpy as np import xarray as xr + return xr.DataArray( + da.ones(shape, dtype=np.uint8, chunks=shape) * values, dims=('y', 'x'), + attrs={'_FillValue': 255, + 'valid_range': [1, 15], + 'name': name, 'area': area, '_satpy_id_keys': local_id_keys_config}) + + +def _create_test_dataset(name, shape=DEFAULT_SHAPE, area=None, values=None): + """Create a test DataArray object.""" + import dask.array as da + import numpy as np + import xarray as xr + + if values: + return xr.DataArray( + da.ones(shape, dtype=np.float32, chunks=shape) * values, dims=('y', 'x'), + attrs={'name': name, 'area': area, '_satpy_id_keys': local_id_keys_config}) + return xr.DataArray( da.zeros(shape, dtype=np.float32, chunks=shape), dims=('y', 'x'), attrs={'name': name, 'area': area, '_satpy_id_keys': local_id_keys_config}) @@ -554,131 +572,220 @@ def test_crop(self): self.assertTupleEqual(new_scn1['4'].shape, (92, 357)) -class TestBlendFuncs(unittest.TestCase): +class TestBlendFuncs(): """Test individual functions used for blending.""" - def setUp(self): - """Set up test data.""" - from datetime import datetime + column = 3 - import dask.array as da - import xarray as xr - from pyresample.geometry import AreaDefinition + @pytest.fixture + def scene1_with_weights(self): + """Create first test scene with a dataset of weights.""" + from satpy import Scene + + area = _create_test_area() + scene = Scene() + dsid1 = make_dataid( + name="geo-ct", + resolution=3000, + modifiers=() + ) + scene[dsid1] = _create_test_int8_dataset(name='geo-ct', area=area, values=1) + wgt1 = _create_test_dataset(name='geo-ct-wgt', area=area, values=0) + + line = 2 + column = 3 + + wgt1[line, :] = 2 + wgt1[:, column] = 2 + + dsid2 = make_dataid( + name="geo-cma", + resolution=3000, + modifiers=() + ) + scene[dsid2] = _create_test_int8_dataset(name='geo-cma', area=area, values=2) + wgt2 = _create_test_dataset(name='geo-cma-wgt', area=area, values=0) + + return scene, [wgt1, wgt2] + + @pytest.fixture + def scene2_with_weights(self): + """Create second test scene.""" + from satpy import Scene + + area = _create_test_area() + scene = Scene() + dsid1 = make_dataid( + name="polar-ct", + modifiers=() + ) + scene[dsid1] = _create_test_int8_dataset(name='polar-ct', area=area, values=3) + scene[dsid1][-1, :] = scene[dsid1].attrs['_FillValue'] + wgt1 = _create_test_dataset(name='polar-ct-wgt', area=area, values=1) + + dsid2 = make_dataid( + name="polar-cma", + modifiers=() + ) + scene[dsid2] = _create_test_int8_dataset(name='polar-cma', area=area, values=4) + wgt2 = _create_test_dataset(name='polar-cma-wgt', area=area, values=1) + return scene, [wgt1, wgt2] + + @pytest.fixture + def multi_scene_and_weights(self, scene1_with_weights, scene2_with_weights): + """Create small multi-scene for testing.""" + from satpy import MultiScene + scene1, weights1 = scene1_with_weights + scene2, weights2 = scene2_with_weights + + return MultiScene([scene1, scene2]), [weights1, weights2] + + @pytest.fixture + def groups(self): + """Get group definitions for the MultiScene.""" + return { + DataQuery(name='CloudType'): ['geo-ct', 'polar-ct'], + DataQuery(name='CloudMask'): ['geo-cma', 'polar-cma'] + } + + def test_blend_two_scenes_using_stack(self, multi_scene_and_weights, groups, + scene1_with_weights, scene2_with_weights): + """Test blending two scenes by stacking them on top of each other using function 'stack'.""" + from satpy.multiscene import stack + + multi_scene, weights = multi_scene_and_weights + scene1, weights1 = scene1_with_weights + scene2, weights2 = scene2_with_weights + + multi_scene.group(groups) + + resampled = multi_scene + stacked = resampled.blend(blend_function=stack) + + expected = scene2['polar-ct'].copy() + expected[-1, :] = scene1['geo-ct'][-1, :] + + xr.testing.assert_equal(stacked['CloudType'].compute(), expected.compute()) + + def test_blend_two_scenes_using_stack_weighted(self, multi_scene_and_weights, groups, + scene1_with_weights, scene2_with_weights): + """Test stacking two scenes using weights.""" + from satpy.multiscene import stack_weighted + + multi_scene, weights = multi_scene_and_weights + scene1, weights1 = scene1_with_weights + scene2, weights2 = scene2_with_weights + + simple_groups = {DataQuery(name='CloudType'): groups[DataQuery(name='CloudType')]} + multi_scene.group(simple_groups) + + weights = [weights[0][0], weights[1][0]] + weighted_blend = multi_scene.blend(blend_function=stack_weighted, weights=weights) + + line = 2 + column = 3 + + expected = scene2['polar-ct'] + expected[line, :] = scene1['geo-ct'][line, :] + expected[:, column] = scene1['geo-ct'][:, column] + expected[-1, :] = scene1['geo-ct'][-1, :] + + result = weighted_blend['CloudType'].compute() + xr.testing.assert_equal(result, expected.compute()) + + @pytest.fixture + def datasets_and_weights(self): + """X-Array datasets with area definition plus weights for input to tests.""" shape = (8, 12) area = AreaDefinition('test', 'test', 'test', {'proj': 'geos', 'lon_0': -95.5, 'h': 35786023.0}, shape[1], shape[0], [-200, -200, 200, 200]) + ds1 = xr.DataArray(da.ones(shape, chunks=-1), dims=('y', 'x'), attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) - self.ds1 = ds1 - wgt1 = xr.DataArray(da.ones(shape, chunks=-1), dims=('y', 'x'), - attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) - self.ds1_wgt = wgt1 ds2 = xr.DataArray(da.ones(shape, chunks=-1) * 2, dims=('y', 'x'), attrs={'start_time': datetime(2018, 1, 1, 1, 0, 0), 'area': area}) - self.ds2 = ds2 - wgt2 = xr.DataArray(da.zeros(shape, chunks=-1), dims=('y', 'x'), - attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) - self.line = 2 - wgt2[self.line, :] = 2 - self.ds2_wgt = wgt2 + ds3 = xr.DataArray(da.ones(shape, chunks=-1) * 3, dims=('y', 'x'), + attrs={'start_time': datetime(2018, 1, 1, 1, 0, 0), 'area': area}) - ds3 = xr.DataArray(da.zeros(shape, chunks=-1), dims=('y', 'time'), - attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) - self.ds3 = ds3 ds4 = xr.DataArray(da.zeros(shape, chunks=-1), dims=('y', 'time'), + attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) + ds5 = xr.DataArray(da.zeros(shape, chunks=-1), dims=('y', 'time'), attrs={'start_time': datetime(2018, 1, 1, 1, 0, 0), 'area': area}) - self.ds4 = ds4 - def test_stack(self): + wgt1 = xr.DataArray(da.ones(shape, chunks=-1), dims=('y', 'x'), + attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) + wgt2 = xr.DataArray(da.zeros(shape, chunks=-1), dims=('y', 'x'), + attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) + wgt3 = xr.DataArray(da.zeros(shape, chunks=-1), dims=('y', 'x'), + attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) + + datastruct = {'shape': shape, + 'area': area, + 'datasets': [ds1, ds2, ds3, ds4, ds5], + 'weights': [wgt1, wgt2, wgt3]} + return datastruct + + @pytest.mark.parametrize(('line', 'column',), + [(2, 3), (4, 5)] + ) + def test_blend_function_stack_weighted(self, datasets_and_weights, line, column): + """Test the 'stack_weighted' function.""" + from satpy.multiscene import stack_weighted + + input_data = datasets_and_weights + + input_data['weights'][1][line, :] = 2 + input_data['weights'][2][:, column] = 2 + + blend_result = stack_weighted(input_data['datasets'][0:3], input_data['weights']) + + ds1 = input_data['datasets'][0] + ds2 = input_data['datasets'][1] + ds3 = input_data['datasets'][2] + expected = ds1.copy() + expected[:, column] = ds3[:, column] + expected[line, :] = ds2[line, :] + + xr.testing.assert_equal(blend_result.compute(), expected.compute()) + assert expected.attrs == blend_result.attrs + + def test_blend_function_stack(self, datasets_and_weights): """Test the 'stack' function.""" from satpy.multiscene import stack - res = stack([self.ds1, self.ds2]) - self.assertTupleEqual(self.ds1.shape, res.shape) - def test_timeseries(self): - """Test the 'timeseries' function.""" - import xarray as xr + input_data = datasets_and_weights + + ds1 = input_data['datasets'][0] + ds2 = input_data['datasets'][1] + + res = stack([ds1, ds2]) + expected = ds2.copy() + xr.testing.assert_equal(res.compute(), expected.compute()) + # assert expected.attrs == res.attrs + # FIXME! Looks like the attributes are taken from the first dataset. Should + # be like that? So in this case the datetime is different from "expected" + # (= in this case the last dataset in the stack, the one on top) + + def test_timeseries(self, datasets_and_weights): + """Test the 'timeseries' function.""" from satpy.multiscene import timeseries - res = timeseries([self.ds1, self.ds2]) - res2 = timeseries([self.ds3, self.ds4]) - self.assertIsInstance(res, xr.DataArray) - self.assertIsInstance(res2, xr.DataArray) - self.assertTupleEqual((2, self.ds1.shape[0], self.ds1.shape[1]), res.shape) - self.assertTupleEqual((self.ds3.shape[0], self.ds3.shape[1]+self.ds4.shape[1]), res2.shape) - - -@pytest.fixture -def datasets_and_weights(): - """X-Array datasets with area definition plus weights for input to tests.""" - shape = (8, 12) - area = AreaDefinition('test', 'test', 'test', - {'proj': 'geos', 'lon_0': -95.5, 'h': 35786023.0}, - shape[1], shape[0], [-200, -200, 200, 200]) - ds1 = xr.DataArray(da.ones(shape, chunks=-1), dims=('y', 'x'), - attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) - ds2 = xr.DataArray(da.ones(shape, chunks=-1) * 2, dims=('y', 'x'), - attrs={'start_time': datetime(2018, 1, 1, 1, 0, 0), 'area': area}) - ds3 = xr.DataArray(da.ones(shape, chunks=-1) * 3, dims=('y', 'x'), - attrs={'start_time': datetime(2018, 1, 1, 1, 0, 0), 'area': area}) - - wgt1 = xr.DataArray(da.ones(shape, chunks=-1), dims=('y', 'x'), - attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) - wgt2 = xr.DataArray(da.zeros(shape, chunks=-1), dims=('y', 'x'), - attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) - wgt3 = xr.DataArray(da.zeros(shape, chunks=-1), dims=('y', 'x'), - attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) - - datastruct = {'shape': shape, - 'area': area, - 'datasets': [ds1, ds2, ds3], - 'weights': [wgt1, wgt2, wgt3]} - return datastruct - - -@pytest.mark.parametrize(('line', 'column',), - [(2, 3), (4, 5)] - ) -def test_blend_function_weighted(datasets_and_weights, line, column): - """Test the 'weighted' function.""" - from satpy.multiscene import weighted - - input_data = datasets_and_weights - - input_data['weights'][1][line, :] = 2 - input_data['weights'][2][:, column] = 2 - - blend_result = weighted(input_data['datasets'], input_data['weights']) - - ds1 = input_data['datasets'][0] - ds2 = input_data['datasets'][1] - ds3 = input_data['datasets'][2] - expected = ds1.copy() - expected[:, column] = ds3[:, column] - expected[line, :] = ds2[line, :] - - xr.testing.assert_equal(blend_result.compute(), expected.compute()) - assert expected.attrs == blend_result.attrs - - -def test_blend_function_stack(datasets_and_weights): - """Test the 'stack' function.""" - from satpy.multiscene import stack - - input_data = datasets_and_weights - - ds1 = input_data['datasets'][0] - ds2 = input_data['datasets'][1] - - res = stack([ds1, ds2]) - expected = ds2.copy() - - xr.testing.assert_equal(res.compute(), expected.compute()) - # assert expected.attrs == res.attrs - # FIXME! Looks like the attributes are taken from the first dataset. Should - # be like that? So in this case the datetime is different from "expected" - # (= in this case the last dataset in the stack, the one on top) + + input_data = datasets_and_weights + + ds1 = input_data['datasets'][0] + ds2 = input_data['datasets'][1] + ds4 = input_data['datasets'][2] + ds4 = input_data['datasets'][3] + ds5 = input_data['datasets'][4] + + res = timeseries([ds1, ds2]) + res2 = timeseries([ds4, ds5]) + assert isinstance(res, xr.DataArray) + assert isinstance(res2, xr.DataArray) + assert (2, ds1.shape[0], ds1.shape[1]) == res.shape + assert (ds4.shape[0], ds4.shape[1]+ds5.shape[1]) == res2.shape @mock.patch('satpy.multiscene.get_enhanced_image') From d4bec0619baa9259dbc49009c69ee0635a7832db Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Thu, 17 Nov 2022 08:53:53 +0100 Subject: [PATCH 04/30] Improve unittest code: Use fixture with autouse to avoid hardcoding column,line Signed-off-by: Adam.Dybbroe --- satpy/tests/test_multiscene.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/satpy/tests/test_multiscene.py b/satpy/tests/test_multiscene.py index deb7cd4bc6..36d8ee488d 100644 --- a/satpy/tests/test_multiscene.py +++ b/satpy/tests/test_multiscene.py @@ -575,7 +575,10 @@ def test_crop(self): class TestBlendFuncs(): """Test individual functions used for blending.""" - column = 3 + @pytest.fixture(autouse=True) + def _get_line_column(self): + self._line = 2 + self._column = 3 @pytest.fixture def scene1_with_weights(self): @@ -592,11 +595,8 @@ def scene1_with_weights(self): scene[dsid1] = _create_test_int8_dataset(name='geo-ct', area=area, values=1) wgt1 = _create_test_dataset(name='geo-ct-wgt', area=area, values=0) - line = 2 - column = 3 - - wgt1[line, :] = 2 - wgt1[:, column] = 2 + wgt1[self._line, :] = 2 + wgt1[:, self._column] = 2 dsid2 = make_dataid( name="geo-cma", @@ -682,12 +682,9 @@ def test_blend_two_scenes_using_stack_weighted(self, multi_scene_and_weights, gr weights = [weights[0][0], weights[1][0]] weighted_blend = multi_scene.blend(blend_function=stack_weighted, weights=weights) - line = 2 - column = 3 - expected = scene2['polar-ct'] - expected[line, :] = scene1['geo-ct'][line, :] - expected[:, column] = scene1['geo-ct'][:, column] + expected[self._line, :] = scene1['geo-ct'][self._line, :] + expected[:, self._column] = scene1['geo-ct'][:, self._column] expected[-1, :] = scene1['geo-ct'][-1, :] result = weighted_blend['CloudType'].compute() From 6a3b52eaf7501fc4d19ac0246920a7e337418395 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Wed, 21 Dec 2022 10:51:21 +0100 Subject: [PATCH 05/30] Change standardname for cloudtype to ct Signed-off-by: Adam.Dybbroe --- satpy/etc/readers/nwcsaf-pps_nc.yaml | 8 ++++---- setup.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/satpy/etc/readers/nwcsaf-pps_nc.yaml b/satpy/etc/readers/nwcsaf-pps_nc.yaml index dd12e1558b..1b673125d6 100644 --- a/satpy/etc/readers/nwcsaf-pps_nc.yaml +++ b/satpy/etc/readers/nwcsaf-pps_nc.yaml @@ -120,25 +120,25 @@ datasets: name: ct file_type: nc_nwcsaf_ct coordinates: [lon, lat] - standard_name: cloudtype + standard_name: ct ct_conditions: name: ct_conditions file_type: nc_nwcsaf_ct coordinates: [lon, lat] - standard_name: cloudtype_conditions + standard_name: ct_conditions ct_quality: name: ct_quality file_type: nc_nwcsaf_ct coordinates: [lon, lat] - standard_name: cloudtype_quality + standard_name: ct_quality ct_status_flag: name: ct_status_flag file_type: nc_nwcsaf_ct coordinates: [lon, lat] - standard_name: cloudtype_status_flag + standard_name: ct_status_flag ct_pal: name: ct_pal diff --git a/setup.py b/setup.py index cdddac0058..cedbb943f1 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2009-2020 Satpy developers +# Copyright (c) 2009-2022 Satpy developers # # This file is part of satpy. # From 51df85e2e3f6fadf726a72b455628b0dd6428b05 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Mon, 23 Jan 2023 12:52:24 +0100 Subject: [PATCH 06/30] Refactor and maintain one stack function available publicly, and make use of partial Signed-off-by: Adam.Dybbroe --- satpy/modifiers/angles.py | 3 ++- satpy/multiscene.py | 41 +++++++++++++++++++++------------- satpy/tests/test_multiscene.py | 16 ++++++++----- 3 files changed, 39 insertions(+), 21 deletions(-) diff --git a/satpy/modifiers/angles.py b/satpy/modifiers/angles.py index 210f9ac37c..9b0f9ea1cf 100644 --- a/satpy/modifiers/angles.py +++ b/satpy/modifiers/angles.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2021 Satpy developers +# Copyright (c) 2021, 2023 Satpy developers # # This file is part of satpy. # @@ -445,6 +445,7 @@ def _get_sensor_angles(data_arr: xr.DataArray) -> tuple[xr.DataArray, xr.DataArr sat_lon, sat_lat, sat_alt = get_satpos(data_arr, preference=preference) area_def = data_arr.attrs["area"] chunks = _geo_chunks_from_data_arr(data_arr) + sata, satz = _get_sensor_angles_from_sat_pos(sat_lon, sat_lat, sat_alt, data_arr.attrs["start_time"], area_def, chunks) diff --git a/satpy/multiscene.py b/satpy/multiscene.py index 89b232380d..37906e53a5 100644 --- a/satpy/multiscene.py +++ b/satpy/multiscene.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2016-2019, 2022 Satpy developers +# Copyright (c) 2016-2019, 2022, 2023 Satpy developers # # This file is part of satpy. # @@ -45,7 +45,29 @@ log = logging.getLogger(__name__) -def stack_weighted(datasets, weights=None): +def stack(datasets, weights=None): + """Combine a series of datasets together. + + On default, datasets are stacked on top of each other, so the last one is + on top. But if a sequence of weights arrays are provided the datasets will + be combined according to those weights. The result will be a composite + dataset where the data in each pixel is coming from the dataset having the + highest weight. + + """ + if weights: + return _stack_weighted(datasets, weights) + + base = datasets[0].copy() + for dataset in datasets[1:]: + try: + base = base.where(dataset == dataset._FillValue, dataset) + except AttributeError: + base = base.where(dataset.isnull(), dataset) + return base + + +def _stack_weighted(datasets, weights): """Stack datasets using weights.""" # Go through weights and set to zero where corresponding datasets have a value equals _FillValue or nan for i, dataset in enumerate(datasets): @@ -62,17 +84,6 @@ def stack_weighted(datasets, weights=None): return weighted_array -def stack(datasets): - """Overlay series of datasets on top of each other.""" - base = datasets[0].copy() - for dataset in datasets[1:]: - try: - base = base.where(dataset == dataset._FillValue, dataset) - except AttributeError: - base = base.where(dataset.isnull(), dataset) - return base - - def timeseries(datasets): """Expand dataset with and concatenate by time dimension.""" expanded_ds = [] @@ -359,7 +370,7 @@ def resample(self, destination=None, **kwargs): """Resample the multiscene.""" return self._generate_scene_func(self._scenes, 'resample', True, destination=destination, **kwargs) - def blend(self, blend_function=stack, **kwargs): + def blend(self, blend_function=stack): """Blend the datasets into one scene. Reduce the :class:`MultiScene` to a single :class:`~satpy.scene.Scene`. Datasets @@ -384,7 +395,7 @@ def blend(self, blend_function=stack, **kwargs): common_datasets = self.shared_dataset_ids for ds_id in common_datasets: datasets = [scn[ds_id] for scn in self.scenes if ds_id in scn] - new_scn[ds_id] = blend_function(datasets, **kwargs) + new_scn[ds_id] = blend_function(datasets) return new_scn diff --git a/satpy/tests/test_multiscene.py b/satpy/tests/test_multiscene.py index 36d8ee488d..0dfc3f0556 100644 --- a/satpy/tests/test_multiscene.py +++ b/satpy/tests/test_multiscene.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2018, 2022 Satpy developers +# Copyright (c) 2018, 2022, 2023 Satpy developers # # This file is part of satpy. # @@ -670,7 +670,9 @@ def test_blend_two_scenes_using_stack(self, multi_scene_and_weights, groups, def test_blend_two_scenes_using_stack_weighted(self, multi_scene_and_weights, groups, scene1_with_weights, scene2_with_weights): """Test stacking two scenes using weights.""" - from satpy.multiscene import stack_weighted + from functools import partial + + from satpy.multiscene import stack multi_scene, weights = multi_scene_and_weights scene1, weights1 = scene1_with_weights @@ -680,7 +682,8 @@ def test_blend_two_scenes_using_stack_weighted(self, multi_scene_and_weights, gr multi_scene.group(simple_groups) weights = [weights[0][0], weights[1][0]] - weighted_blend = multi_scene.blend(blend_function=stack_weighted, weights=weights) + stack_with_weights = partial(stack, weights=weights) + weighted_blend = multi_scene.blend(blend_function=stack_with_weights) expected = scene2['polar-ct'] expected[self._line, :] = scene1['geo-ct'][self._line, :] @@ -728,14 +731,17 @@ def datasets_and_weights(self): ) def test_blend_function_stack_weighted(self, datasets_and_weights, line, column): """Test the 'stack_weighted' function.""" - from satpy.multiscene import stack_weighted + from functools import partial + + from satpy.multiscene import stack input_data = datasets_and_weights input_data['weights'][1][line, :] = 2 input_data['weights'][2][:, column] = 2 - blend_result = stack_weighted(input_data['datasets'][0:3], input_data['weights']) + stack_with_weights = partial(stack, weights=input_data['weights']) + blend_result = stack_with_weights(input_data['datasets'][0:3]) ds1 = input_data['datasets'][0] ds2 = input_data['datasets'][1] From d32eeebbce0baa36182bfc99e4cd8d1c105d5f6d Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Mon, 23 Jan 2023 14:17:00 +0100 Subject: [PATCH 07/30] Use setup method rather than autouse fixture Signed-off-by: Adam.Dybbroe --- satpy/tests/test_multiscene.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/tests/test_multiscene.py b/satpy/tests/test_multiscene.py index 0dfc3f0556..fdf571e51b 100644 --- a/satpy/tests/test_multiscene.py +++ b/satpy/tests/test_multiscene.py @@ -575,8 +575,8 @@ def test_crop(self): class TestBlendFuncs(): """Test individual functions used for blending.""" - @pytest.fixture(autouse=True) - def _get_line_column(self): + def setup_method(self): + """Set up test functions.""" self._line = 2 self._column = 3 From 1abfc5a785ea1382dc2148ca43f35b73e95ee819 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Mon, 23 Jan 2023 19:44:31 +0100 Subject: [PATCH 08/30] Re-arrange multiscene tests into three modules in a sub-directory Signed-off-by: Adam.Dybbroe --- satpy/tests/multiscene_tests/__init__.py | 120 +++ satpy/tests/multiscene_tests/test_blend.py | 249 +++++ satpy/tests/multiscene_tests/test_misc.py | 204 +++++ .../multiscene_tests/test_save_animation.py | 368 ++++++++ satpy/tests/test_multiscene.py | 866 ------------------ 5 files changed, 941 insertions(+), 866 deletions(-) create mode 100644 satpy/tests/multiscene_tests/__init__.py create mode 100644 satpy/tests/multiscene_tests/test_blend.py create mode 100644 satpy/tests/multiscene_tests/test_misc.py create mode 100644 satpy/tests/multiscene_tests/test_save_animation.py delete mode 100644 satpy/tests/test_multiscene.py diff --git a/satpy/tests/multiscene_tests/__init__.py b/satpy/tests/multiscene_tests/__init__.py new file mode 100644 index 0000000000..f2a27534be --- /dev/null +++ b/satpy/tests/multiscene_tests/__init__.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2018-2023 Satpy developers +# +# This file is part of satpy. +# +# satpy is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# satpy is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# satpy. If not, see . +"""Unit tests for Multiscene.""" + +# NOTE: +# The following fixtures are not defined in this file, but are used and injected by Pytest: +# - tmp_path + +import dask.array as da +import numpy as np +import xarray as xr +from pyresample.geometry import AreaDefinition + +from satpy.dataset.dataid import DataID, ModifierTuple, WavelengthRange + +DEFAULT_SHAPE = (5, 10) + +local_id_keys_config = {'name': { + 'required': True, +}, + 'wavelength': { + 'type': WavelengthRange, +}, + 'resolution': None, + 'calibration': { + 'enum': [ + 'reflectance', + 'brightness_temperature', + 'radiance', + 'counts' + ] +}, + 'polarization': None, + 'level': None, + 'modifiers': { + 'required': True, + 'default': ModifierTuple(), + 'type': ModifierTuple, +}, +} + + +def make_dataid(**items): + """Make a data id.""" + return DataID(local_id_keys_config, **items) + + +def _fake_get_enhanced_image(img, enhance=None, overlay=None, decorate=None): + from trollimage.xrimage import XRImage + return XRImage(img) + + +def _create_test_area(proj_str=None, shape=DEFAULT_SHAPE, extents=None): + """Create a test area definition.""" + from pyresample.utils import proj4_str_to_dict + if proj_str is None: + proj_str = '+proj=lcc +datum=WGS84 +ellps=WGS84 +lon_0=-95. ' \ + '+lat_0=25 +lat_1=25 +units=m +no_defs' + proj_dict = proj4_str_to_dict(proj_str) + extents = extents or (-1000., -1500., 1000., 1500.) + + return AreaDefinition( + 'test', + 'test', + 'test', + proj_dict, + shape[1], + shape[0], + extents + ) + + +def _create_test_int8_dataset(name, shape=DEFAULT_SHAPE, area=None, values=None): + """Create a test DataArray object.""" + return xr.DataArray( + da.ones(shape, dtype=np.uint8, chunks=shape) * values, dims=('y', 'x'), + attrs={'_FillValue': 255, + 'valid_range': [1, 15], + 'name': name, 'area': area, '_satpy_id_keys': local_id_keys_config}) + + +def _create_test_dataset(name, shape=DEFAULT_SHAPE, area=None, values=None): + """Create a test DataArray object.""" + if values: + return xr.DataArray( + da.ones(shape, dtype=np.float32, chunks=shape) * values, dims=('y', 'x'), + attrs={'name': name, 'area': area, '_satpy_id_keys': local_id_keys_config}) + + return xr.DataArray( + da.zeros(shape, dtype=np.float32, chunks=shape), dims=('y', 'x'), + attrs={'name': name, 'area': area, '_satpy_id_keys': local_id_keys_config}) + + +def _create_test_scenes(num_scenes=2, shape=DEFAULT_SHAPE, area=None): + """Create some test scenes for various test cases.""" + from satpy import Scene + ds1 = _create_test_dataset('ds1', shape=shape, area=area) + ds2 = _create_test_dataset('ds2', shape=shape, area=area) + scenes = [] + for _ in range(num_scenes): + scn = Scene() + scn['ds1'] = ds1.copy() + scn['ds2'] = ds2.copy() + scenes.append(scn) + return scenes diff --git a/satpy/tests/multiscene_tests/test_blend.py b/satpy/tests/multiscene_tests/test_blend.py new file mode 100644 index 0000000000..bf3e96d711 --- /dev/null +++ b/satpy/tests/multiscene_tests/test_blend.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2018-2023 Satpy developers +# +# This file is part of satpy. +# +# satpy is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# satpy is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# satpy. If not, see . + +"""Unit tests for blending datasets with the Multiscene object.""" + +from datetime import datetime + +import dask.array as da +import pytest +import xarray as xr +from pyresample.geometry import AreaDefinition + +from satpy import DataQuery +from satpy.tests.multiscene_tests import _create_test_area, _create_test_dataset, _create_test_int8_dataset, make_dataid + + +class TestBlendFuncs(): + """Test individual functions used for blending.""" + + def setup_method(self): + """Set up test functions.""" + self._line = 2 + self._column = 3 + + @pytest.fixture + def scene1_with_weights(self): + """Create first test scene with a dataset of weights.""" + from satpy import Scene + + area = _create_test_area() + scene = Scene() + dsid1 = make_dataid( + name="geo-ct", + resolution=3000, + modifiers=() + ) + scene[dsid1] = _create_test_int8_dataset(name='geo-ct', area=area, values=1) + wgt1 = _create_test_dataset(name='geo-ct-wgt', area=area, values=0) + + wgt1[self._line, :] = 2 + wgt1[:, self._column] = 2 + + dsid2 = make_dataid( + name="geo-cma", + resolution=3000, + modifiers=() + ) + scene[dsid2] = _create_test_int8_dataset(name='geo-cma', area=area, values=2) + wgt2 = _create_test_dataset(name='geo-cma-wgt', area=area, values=0) + + return scene, [wgt1, wgt2] + + @pytest.fixture + def scene2_with_weights(self): + """Create second test scene.""" + from satpy import Scene + + area = _create_test_area() + scene = Scene() + dsid1 = make_dataid( + name="polar-ct", + modifiers=() + ) + scene[dsid1] = _create_test_int8_dataset(name='polar-ct', area=area, values=3) + scene[dsid1][-1, :] = scene[dsid1].attrs['_FillValue'] + wgt1 = _create_test_dataset(name='polar-ct-wgt', area=area, values=1) + + dsid2 = make_dataid( + name="polar-cma", + modifiers=() + ) + scene[dsid2] = _create_test_int8_dataset(name='polar-cma', area=area, values=4) + wgt2 = _create_test_dataset(name='polar-cma-wgt', area=area, values=1) + return scene, [wgt1, wgt2] + + @pytest.fixture + def multi_scene_and_weights(self, scene1_with_weights, scene2_with_weights): + """Create small multi-scene for testing.""" + from satpy import MultiScene + scene1, weights1 = scene1_with_weights + scene2, weights2 = scene2_with_weights + + return MultiScene([scene1, scene2]), [weights1, weights2] + + @pytest.fixture + def groups(self): + """Get group definitions for the MultiScene.""" + return { + DataQuery(name='CloudType'): ['geo-ct', 'polar-ct'], + DataQuery(name='CloudMask'): ['geo-cma', 'polar-cma'] + } + + def test_blend_two_scenes_using_stack(self, multi_scene_and_weights, groups, + scene1_with_weights, scene2_with_weights): + """Test blending two scenes by stacking them on top of each other using function 'stack'.""" + from satpy.multiscene import stack + + multi_scene, weights = multi_scene_and_weights + scene1, weights1 = scene1_with_weights + scene2, weights2 = scene2_with_weights + + multi_scene.group(groups) + + resampled = multi_scene + stacked = resampled.blend(blend_function=stack) + + expected = scene2['polar-ct'].copy() + expected[-1, :] = scene1['geo-ct'][-1, :] + + xr.testing.assert_equal(stacked['CloudType'].compute(), expected.compute()) + + def test_blend_two_scenes_using_stack_weighted(self, multi_scene_and_weights, groups, + scene1_with_weights, scene2_with_weights): + """Test stacking two scenes using weights.""" + from functools import partial + + from satpy.multiscene import stack + + multi_scene, weights = multi_scene_and_weights + scene1, weights1 = scene1_with_weights + scene2, weights2 = scene2_with_weights + + simple_groups = {DataQuery(name='CloudType'): groups[DataQuery(name='CloudType')]} + multi_scene.group(simple_groups) + + weights = [weights[0][0], weights[1][0]] + stack_with_weights = partial(stack, weights=weights) + weighted_blend = multi_scene.blend(blend_function=stack_with_weights) + + expected = scene2['polar-ct'] + expected[self._line, :] = scene1['geo-ct'][self._line, :] + expected[:, self._column] = scene1['geo-ct'][:, self._column] + expected[-1, :] = scene1['geo-ct'][-1, :] + + result = weighted_blend['CloudType'].compute() + xr.testing.assert_equal(result, expected.compute()) + + @pytest.fixture + def datasets_and_weights(self): + """X-Array datasets with area definition plus weights for input to tests.""" + shape = (8, 12) + area = AreaDefinition('test', 'test', 'test', + {'proj': 'geos', 'lon_0': -95.5, 'h': 35786023.0}, + shape[1], shape[0], [-200, -200, 200, 200]) + + ds1 = xr.DataArray(da.ones(shape, chunks=-1), dims=('y', 'x'), + attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) + ds2 = xr.DataArray(da.ones(shape, chunks=-1) * 2, dims=('y', 'x'), + attrs={'start_time': datetime(2018, 1, 1, 1, 0, 0), 'area': area}) + ds3 = xr.DataArray(da.ones(shape, chunks=-1) * 3, dims=('y', 'x'), + attrs={'start_time': datetime(2018, 1, 1, 1, 0, 0), 'area': area}) + + ds4 = xr.DataArray(da.zeros(shape, chunks=-1), dims=('y', 'time'), + attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) + ds5 = xr.DataArray(da.zeros(shape, chunks=-1), dims=('y', 'time'), + attrs={'start_time': datetime(2018, 1, 1, 1, 0, 0), 'area': area}) + + wgt1 = xr.DataArray(da.ones(shape, chunks=-1), dims=('y', 'x'), + attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) + wgt2 = xr.DataArray(da.zeros(shape, chunks=-1), dims=('y', 'x'), + attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) + wgt3 = xr.DataArray(da.zeros(shape, chunks=-1), dims=('y', 'x'), + attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) + + datastruct = {'shape': shape, + 'area': area, + 'datasets': [ds1, ds2, ds3, ds4, ds5], + 'weights': [wgt1, wgt2, wgt3]} + return datastruct + + @pytest.mark.parametrize(('line', 'column',), + [(2, 3), (4, 5)] + ) + def test_blend_function_stack_weighted(self, datasets_and_weights, line, column): + """Test the 'stack_weighted' function.""" + from functools import partial + + from satpy.multiscene import stack + + input_data = datasets_and_weights + + input_data['weights'][1][line, :] = 2 + input_data['weights'][2][:, column] = 2 + + stack_with_weights = partial(stack, weights=input_data['weights']) + blend_result = stack_with_weights(input_data['datasets'][0:3]) + + ds1 = input_data['datasets'][0] + ds2 = input_data['datasets'][1] + ds3 = input_data['datasets'][2] + expected = ds1.copy() + expected[:, column] = ds3[:, column] + expected[line, :] = ds2[line, :] + + xr.testing.assert_equal(blend_result.compute(), expected.compute()) + assert expected.attrs == blend_result.attrs + + def test_blend_function_stack(self, datasets_and_weights): + """Test the 'stack' function.""" + from satpy.multiscene import stack + + input_data = datasets_and_weights + + ds1 = input_data['datasets'][0] + ds2 = input_data['datasets'][1] + + res = stack([ds1, ds2]) + expected = ds2.copy() + + xr.testing.assert_equal(res.compute(), expected.compute()) + # assert expected.attrs == res.attrs + # FIXME! Looks like the attributes are taken from the first dataset. Should + # be like that? So in this case the datetime is different from "expected" + # (= in this case the last dataset in the stack, the one on top) + + def test_timeseries(self, datasets_and_weights): + """Test the 'timeseries' function.""" + from satpy.multiscene import timeseries + + input_data = datasets_and_weights + + ds1 = input_data['datasets'][0] + ds2 = input_data['datasets'][1] + ds4 = input_data['datasets'][2] + ds4 = input_data['datasets'][3] + ds5 = input_data['datasets'][4] + + res = timeseries([ds1, ds2]) + res2 = timeseries([ds4, ds5]) + assert isinstance(res, xr.DataArray) + assert isinstance(res2, xr.DataArray) + assert (2, ds1.shape[0], ds1.shape[1]) == res.shape + assert (ds4.shape[0], ds4.shape[1]+ds5.shape[1]) == res2.shape diff --git a/satpy/tests/multiscene_tests/test_misc.py b/satpy/tests/multiscene_tests/test_misc.py new file mode 100644 index 0000000000..62340ff0ad --- /dev/null +++ b/satpy/tests/multiscene_tests/test_misc.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2018-2023 Satpy developers +# +# This file is part of satpy. +# +# satpy is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# satpy is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# satpy. If not, see . + +"""Unit tests for the Multiscene object.""" + +import unittest +from unittest import mock + +import pytest +import xarray as xr + +from satpy import DataQuery +from satpy.tests.multiscene_tests import _create_test_area, _create_test_dataset, _create_test_scenes, make_dataid + + +class TestMultiScene(unittest.TestCase): + """Test basic functionality of MultiScene.""" + + def test_init_empty(self): + """Test creating a multiscene with no children.""" + from satpy import MultiScene + MultiScene() + + def test_init_children(self): + """Test creating a multiscene with children.""" + from satpy import MultiScene + scenes = _create_test_scenes() + MultiScene(scenes) + + def test_properties(self): + """Test basic properties/attributes of the MultiScene.""" + from satpy import MultiScene + + area = _create_test_area() + scenes = _create_test_scenes(area=area) + ds1_id = make_dataid(name='ds1') + ds2_id = make_dataid(name='ds2') + ds3_id = make_dataid(name='ds3') + ds4_id = make_dataid(name='ds4') + + # Add a dataset to only one of the Scenes + scenes[1]['ds3'] = _create_test_dataset('ds3') + mscn = MultiScene(scenes) + + self.assertSetEqual(mscn.loaded_dataset_ids, + {ds1_id, ds2_id, ds3_id}) + self.assertSetEqual(mscn.shared_dataset_ids, {ds1_id, ds2_id}) + self.assertTrue(mscn.all_same_area) + + bigger_area = _create_test_area(shape=(20, 40)) + scenes[0]['ds4'] = _create_test_dataset('ds4', shape=(20, 40), + area=bigger_area) + + self.assertSetEqual(mscn.loaded_dataset_ids, + {ds1_id, ds2_id, ds3_id, ds4_id}) + self.assertSetEqual(mscn.shared_dataset_ids, {ds1_id, ds2_id}) + self.assertFalse(mscn.all_same_area) + + def test_from_files(self): + """Test creating a multiscene from multiple files.""" + from satpy import MultiScene + input_files_abi = [ + "OR_ABI-L1b-RadC-M3C01_G16_s20171171502203_e20171171504576_c20171171505018.nc", + "OR_ABI-L1b-RadC-M3C01_G16_s20171171507203_e20171171509576_c20171171510018.nc", + "OR_ABI-L1b-RadC-M3C01_G16_s20171171512203_e20171171514576_c20171171515017.nc", + "OR_ABI-L1b-RadC-M3C01_G16_s20171171517203_e20171171519577_c20171171520019.nc", + "OR_ABI-L1b-RadC-M3C01_G16_s20171171522203_e20171171524576_c20171171525020.nc", + "OR_ABI-L1b-RadC-M3C01_G16_s20171171527203_e20171171529576_c20171171530017.nc", + ] + input_files_glm = [ + "OR_GLM-L2-GLMC-M3_G16_s20171171500000_e20171171501000_c20380190314080.nc", + "OR_GLM-L2-GLMC-M3_G16_s20171171501000_e20171171502000_c20380190314080.nc", + "OR_GLM-L2-GLMC-M3_G16_s20171171502000_e20171171503000_c20380190314080.nc", + "OR_GLM-L2-GLMC-M3_G16_s20171171503000_e20171171504000_c20380190314080.nc", + "OR_GLM-L2-GLMC-M3_G16_s20171171504000_e20171171505000_c20380190314080.nc", + "OR_GLM-L2-GLMC-M3_G16_s20171171505000_e20171171506000_c20380190314080.nc", + "OR_GLM-L2-GLMC-M3_G16_s20171171506000_e20171171507000_c20380190314080.nc", + "OR_GLM-L2-GLMC-M3_G16_s20171171507000_e20171171508000_c20380190314080.nc", + ] + with mock.patch('satpy.multiscene.Scene') as scn_mock: + mscn = MultiScene.from_files( + input_files_abi, + reader='abi_l1b', + scene_kwargs={"reader_kwargs": {}}) + assert len(mscn.scenes) == 6 + calls = [mock.call( + filenames={'abi_l1b': [in_file_abi]}, + reader_kwargs={}) + for in_file_abi in input_files_abi] + scn_mock.assert_has_calls(calls) + + scn_mock.reset_mock() + with pytest.warns(DeprecationWarning): + mscn = MultiScene.from_files( + input_files_abi + input_files_glm, + reader=('abi_l1b', "glm_l2"), + group_keys=["start_time"], + ensure_all_readers=True, + time_threshold=30) + assert len(mscn.scenes) == 2 + calls = [mock.call( + filenames={'abi_l1b': [in_file_abi], 'glm_l2': [in_file_glm]}) + for (in_file_abi, in_file_glm) in + zip(input_files_abi[0:2], + [input_files_glm[2]] + [input_files_glm[7]])] + scn_mock.assert_has_calls(calls) + scn_mock.reset_mock() + mscn = MultiScene.from_files( + input_files_abi + input_files_glm, + reader=('abi_l1b', "glm_l2"), + group_keys=["start_time"], + ensure_all_readers=False, + time_threshold=30) + assert len(mscn.scenes) == 12 + + +class TestMultiSceneGrouping: + """Test dataset grouping in MultiScene.""" + + @pytest.fixture + def scene1(self): + """Create first test scene.""" + from satpy import Scene + scene = Scene() + dsid1 = make_dataid( + name="ds1", + resolution=123, + wavelength=(1, 2, 3), + polarization="H" + ) + scene[dsid1] = _create_test_dataset(name='ds1') + dsid2 = make_dataid( + name="ds2", + resolution=456, + wavelength=(4, 5, 6), + polarization="V" + ) + scene[dsid2] = _create_test_dataset(name='ds2') + return scene + + @pytest.fixture + def scene2(self): + """Create second test scene.""" + from satpy import Scene + scene = Scene() + dsid1 = make_dataid( + name="ds3", + resolution=123.1, + wavelength=(1.1, 2.1, 3.1), + polarization="H" + ) + scene[dsid1] = _create_test_dataset(name='ds3') + dsid2 = make_dataid( + name="ds4", + resolution=456.1, + wavelength=(4.1, 5.1, 6.1), + polarization="V" + ) + scene[dsid2] = _create_test_dataset(name='ds4') + return scene + + @pytest.fixture + def multi_scene(self, scene1, scene2): + """Create small multi scene for testing.""" + from satpy import MultiScene + return MultiScene([scene1, scene2]) + + @pytest.fixture + def groups(self): + """Get group definitions for the MultiScene.""" + return { + DataQuery(name='odd'): ['ds1', 'ds3'], + DataQuery(name='even'): ['ds2', 'ds4'] + } + + def test_multi_scene_grouping(self, multi_scene, groups, scene1): + """Test grouping a MultiScene.""" + multi_scene.group(groups) + shared_ids_exp = {make_dataid(name="odd"), make_dataid(name="even")} + assert multi_scene.shared_dataset_ids == shared_ids_exp + assert DataQuery(name='odd') not in scene1 + xr.testing.assert_allclose(multi_scene.scenes[0]["ds1"], scene1["ds1"]) + + def test_fails_to_add_multiple_datasets_from_the_same_scene_to_a_group(self, multi_scene): + """Test that multiple datasets from the same scene in one group fails.""" + groups = {DataQuery(name='mygroup'): ['ds1', 'ds2']} + multi_scene.group(groups) + with pytest.raises(ValueError): + next(multi_scene.scenes) diff --git a/satpy/tests/multiscene_tests/test_save_animation.py b/satpy/tests/multiscene_tests/test_save_animation.py new file mode 100644 index 0000000000..6b62033059 --- /dev/null +++ b/satpy/tests/multiscene_tests/test_save_animation.py @@ -0,0 +1,368 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2018-2023 Satpy developers +# +# This file is part of satpy. +# +# satpy is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# satpy is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# satpy. If not, see . + +"""Unit tests for saving animations using Multiscene.""" + +import os +import shutil +import tempfile +import unittest +from datetime import datetime +from unittest import mock + +from satpy.tests.multiscene_tests import ( + _create_test_area, + _create_test_dataset, + _create_test_scenes, + _fake_get_enhanced_image, +) + + +class TestMultiSceneSave(unittest.TestCase): + """Test saving a MultiScene to various formats.""" + + def setUp(self): + """Create temporary directory to save files to.""" + self.base_dir = tempfile.mkdtemp() + + def tearDown(self): + """Remove the temporary directory created for a test.""" + try: + shutil.rmtree(self.base_dir, ignore_errors=True) + except OSError: + pass + + @mock.patch('satpy.multiscene.get_enhanced_image', _fake_get_enhanced_image) + def test_save_mp4_distributed(self): + """Save a series of fake scenes to an mp4 video.""" + from satpy import MultiScene + area = _create_test_area() + scenes = _create_test_scenes(area=area) + + # Add a dataset to only one of the Scenes + scenes[1]['ds3'] = _create_test_dataset('ds3') + # Add a start and end time + for ds_id in ['ds1', 'ds2', 'ds3']: + scenes[1][ds_id].attrs['start_time'] = datetime(2018, 1, 2) + scenes[1][ds_id].attrs['end_time'] = datetime(2018, 1, 2, 12) + if ds_id == 'ds3': + continue + scenes[0][ds_id].attrs['start_time'] = datetime(2018, 1, 1) + scenes[0][ds_id].attrs['end_time'] = datetime(2018, 1, 1, 12) + + mscn = MultiScene(scenes) + fn = os.path.join( + self.base_dir, + 'test_save_mp4_{name}_{start_time:%Y%m%d_%H}_{end_time:%Y%m%d_%H}.mp4') + writer_mock = mock.MagicMock() + client_mock = mock.MagicMock() + client_mock.compute.side_effect = lambda x: tuple(v.compute() for v in x) + client_mock.gather.side_effect = lambda x: x + with mock.patch('satpy.multiscene.imageio.get_writer') as get_writer: + get_writer.return_value = writer_mock + # force order of datasets by specifying them + mscn.save_animation(fn, client=client_mock, datasets=['ds1', 'ds2', 'ds3']) + + # 2 saves for the first scene + 1 black frame + # 3 for the second scene + self.assertEqual(writer_mock.append_data.call_count, 3 + 3) + filenames = [os.path.basename(args[0][0]) for args in get_writer.call_args_list] + self.assertEqual(filenames[0], 'test_save_mp4_ds1_20180101_00_20180102_12.mp4') + self.assertEqual(filenames[1], 'test_save_mp4_ds2_20180101_00_20180102_12.mp4') + self.assertEqual(filenames[2], 'test_save_mp4_ds3_20180102_00_20180102_12.mp4') + + # Test no distributed client found + mscn = MultiScene(scenes) + fn = os.path.join( + self.base_dir, + 'test_save_mp4_{name}_{start_time:%Y%m%d_%H}_{end_time:%Y%m%d_%H}.mp4') + writer_mock = mock.MagicMock() + client_mock = mock.MagicMock() + client_mock.compute.side_effect = lambda x: tuple(v.compute() for v in x) + client_mock.gather.side_effect = lambda x: x + with mock.patch('satpy.multiscene.imageio.get_writer') as get_writer, \ + mock.patch('satpy.multiscene.get_client', mock.Mock(side_effect=ValueError("No client"))): + get_writer.return_value = writer_mock + # force order of datasets by specifying them + mscn.save_animation(fn, datasets=['ds1', 'ds2', 'ds3']) + + # 2 saves for the first scene + 1 black frame + # 3 for the second scene + self.assertEqual(writer_mock.append_data.call_count, 3 + 3) + filenames = [os.path.basename(args[0][0]) for args in get_writer.call_args_list] + self.assertEqual(filenames[0], 'test_save_mp4_ds1_20180101_00_20180102_12.mp4') + self.assertEqual(filenames[1], 'test_save_mp4_ds2_20180101_00_20180102_12.mp4') + self.assertEqual(filenames[2], 'test_save_mp4_ds3_20180102_00_20180102_12.mp4') + + @mock.patch('satpy.multiscene.get_enhanced_image', _fake_get_enhanced_image) + def test_save_mp4_no_distributed(self): + """Save a series of fake scenes to an mp4 video when distributed isn't available.""" + from satpy import MultiScene + area = _create_test_area() + scenes = _create_test_scenes(area=area) + + # Add a dataset to only one of the Scenes + scenes[1]['ds3'] = _create_test_dataset('ds3') + # Add a start and end time + for ds_id in ['ds1', 'ds2', 'ds3']: + scenes[1][ds_id].attrs['start_time'] = datetime(2018, 1, 2) + scenes[1][ds_id].attrs['end_time'] = datetime(2018, 1, 2, 12) + if ds_id == 'ds3': + continue + scenes[0][ds_id].attrs['start_time'] = datetime(2018, 1, 1) + scenes[0][ds_id].attrs['end_time'] = datetime(2018, 1, 1, 12) + + mscn = MultiScene(scenes) + fn = os.path.join( + self.base_dir, + 'test_save_mp4_{name}_{start_time:%Y%m%d_%H}_{end_time:%Y%m%d_%H}.mp4') + writer_mock = mock.MagicMock() + client_mock = mock.MagicMock() + client_mock.compute.side_effect = lambda x: tuple(v.compute() for v in x) + client_mock.gather.side_effect = lambda x: x + with mock.patch('satpy.multiscene.imageio.get_writer') as get_writer, \ + mock.patch('satpy.multiscene.get_client', None): + get_writer.return_value = writer_mock + # force order of datasets by specifying them + mscn.save_animation(fn, datasets=['ds1', 'ds2', 'ds3']) + + # 2 saves for the first scene + 1 black frame + # 3 for the second scene + self.assertEqual(writer_mock.append_data.call_count, 3 + 3) + filenames = [os.path.basename(args[0][0]) for args in get_writer.call_args_list] + self.assertEqual(filenames[0], 'test_save_mp4_ds1_20180101_00_20180102_12.mp4') + self.assertEqual(filenames[1], 'test_save_mp4_ds2_20180101_00_20180102_12.mp4') + self.assertEqual(filenames[2], 'test_save_mp4_ds3_20180102_00_20180102_12.mp4') + + @mock.patch('satpy.multiscene.get_enhanced_image', _fake_get_enhanced_image) + def test_save_datasets_simple(self): + """Save a series of fake scenes to an PNG images.""" + from satpy import MultiScene + area = _create_test_area() + scenes = _create_test_scenes(area=area) + + # Add a dataset to only one of the Scenes + scenes[1]['ds3'] = _create_test_dataset('ds3') + # Add a start and end time + for ds_id in ['ds1', 'ds2', 'ds3']: + scenes[1][ds_id].attrs['start_time'] = datetime(2018, 1, 2) + scenes[1][ds_id].attrs['end_time'] = datetime(2018, 1, 2, 12) + if ds_id == 'ds3': + continue + scenes[0][ds_id].attrs['start_time'] = datetime(2018, 1, 1) + scenes[0][ds_id].attrs['end_time'] = datetime(2018, 1, 1, 12) + + mscn = MultiScene(scenes) + client_mock = mock.MagicMock() + client_mock.compute.side_effect = lambda x: tuple(v for v in x) + client_mock.gather.side_effect = lambda x: x + with mock.patch('satpy.multiscene.Scene.save_datasets') as save_datasets: + save_datasets.return_value = [True] # some arbitrary return value + # force order of datasets by specifying them + mscn.save_datasets(base_dir=self.base_dir, client=False, datasets=['ds1', 'ds2', 'ds3'], + writer='simple_image') + + # 2 for each scene + self.assertEqual(save_datasets.call_count, 2) + + @mock.patch('satpy.multiscene.get_enhanced_image', _fake_get_enhanced_image) + def test_save_datasets_distributed_delayed(self): + """Test distributed save for writers returning delayed obejcts e.g. simple_image.""" + from dask.delayed import Delayed + + from satpy import MultiScene + area = _create_test_area() + scenes = _create_test_scenes(area=area) + + # Add a dataset to only one of the Scenes + scenes[1]['ds3'] = _create_test_dataset('ds3') + # Add a start and end time + for ds_id in ['ds1', 'ds2', 'ds3']: + scenes[1][ds_id].attrs['start_time'] = datetime(2018, 1, 2) + scenes[1][ds_id].attrs['end_time'] = datetime(2018, 1, 2, 12) + if ds_id == 'ds3': + continue + scenes[0][ds_id].attrs['start_time'] = datetime(2018, 1, 1) + scenes[0][ds_id].attrs['end_time'] = datetime(2018, 1, 1, 12) + + mscn = MultiScene(scenes) + client_mock = mock.MagicMock() + client_mock.compute.side_effect = lambda x: tuple(v for v in x) + client_mock.gather.side_effect = lambda x: x + future_mock = mock.MagicMock() + future_mock.__class__ = Delayed + with mock.patch('satpy.multiscene.Scene.save_datasets') as save_datasets: + save_datasets.return_value = [future_mock] # some arbitrary return value + # force order of datasets by specifying them + mscn.save_datasets(base_dir=self.base_dir, client=client_mock, datasets=['ds1', 'ds2', 'ds3'], + writer='simple_image') + + # 2 for each scene + self.assertEqual(save_datasets.call_count, 2) + + @mock.patch('satpy.multiscene.get_enhanced_image', _fake_get_enhanced_image) + def test_save_datasets_distributed_source_target(self): + """Test distributed save for writers returning sources and targets e.g. geotiff writer.""" + import dask.array as da + + from satpy import MultiScene + area = _create_test_area() + scenes = _create_test_scenes(area=area) + + # Add a dataset to only one of the Scenes + scenes[1]['ds3'] = _create_test_dataset('ds3') + # Add a start and end time + for ds_id in ['ds1', 'ds2', 'ds3']: + scenes[1][ds_id].attrs['start_time'] = datetime(2018, 1, 2) + scenes[1][ds_id].attrs['end_time'] = datetime(2018, 1, 2, 12) + if ds_id == 'ds3': + continue + scenes[0][ds_id].attrs['start_time'] = datetime(2018, 1, 1) + scenes[0][ds_id].attrs['end_time'] = datetime(2018, 1, 1, 12) + + mscn = MultiScene(scenes) + client_mock = mock.MagicMock() + client_mock.compute.side_effect = lambda x: tuple(v for v in x) + client_mock.gather.side_effect = lambda x: x + source_mock = mock.MagicMock() + source_mock.__class__ = da.Array + target_mock = mock.MagicMock() + with mock.patch('satpy.multiscene.Scene.save_datasets') as save_datasets: + save_datasets.return_value = [(source_mock, target_mock)] # some arbitrary return value + # force order of datasets by specifying them + with self.assertRaises(NotImplementedError): + mscn.save_datasets(base_dir=self.base_dir, client=client_mock, datasets=['ds1', 'ds2', 'ds3'], + writer='geotiff') + + def test_crop(self): + """Test the crop method.""" + import numpy as np + from pyresample.geometry import AreaDefinition + from xarray import DataArray + + from satpy import MultiScene, Scene + scene1 = Scene() + area_extent = (-5570248.477339745, -5561247.267842293, 5567248.074173927, + 5570248.477339745) + proj_dict = {'a': 6378169.0, 'b': 6356583.8, 'h': 35785831.0, + 'lon_0': 0.0, 'proj': 'geos', 'units': 'm'} + x_size = 3712 + y_size = 3712 + area_def = AreaDefinition( + 'test', 'test', 'test', + proj_dict, + x_size, + y_size, + area_extent, + ) + area_def2 = AreaDefinition( + 'test2', 'test2', 'test2', proj_dict, + x_size // 2, + y_size // 2, + area_extent, + ) + scene1["1"] = DataArray(np.zeros((y_size, x_size))) + scene1["2"] = DataArray(np.zeros((y_size, x_size)), dims=('y', 'x')) + scene1["3"] = DataArray(np.zeros((y_size, x_size)), dims=('y', 'x'), + attrs={'area': area_def}) + scene1["4"] = DataArray(np.zeros((y_size // 2, x_size // 2)), dims=('y', 'x'), + attrs={'area': area_def2}) + mscn = MultiScene([scene1]) + + # by lon/lat bbox + new_mscn = mscn.crop(ll_bbox=(-20., -5., 0, 0)) + new_scn1 = list(new_mscn.scenes)[0] + self.assertIn('1', new_scn1) + self.assertIn('2', new_scn1) + self.assertIn('3', new_scn1) + self.assertTupleEqual(new_scn1['1'].shape, (y_size, x_size)) + self.assertTupleEqual(new_scn1['2'].shape, (y_size, x_size)) + self.assertTupleEqual(new_scn1['3'].shape, (184, 714)) + self.assertTupleEqual(new_scn1['4'].shape, (92, 357)) + + +@mock.patch('satpy.multiscene.get_enhanced_image') +def test_save_mp4(smg, tmp_path): + """Save a series of fake scenes to an mp4 video.""" + from satpy import MultiScene + area = _create_test_area() + scenes = _create_test_scenes(area=area) + smg.side_effect = _fake_get_enhanced_image + + # Add a dataset to only one of the Scenes + scenes[1]['ds3'] = _create_test_dataset('ds3') + # Add a start and end time + for ds_id in ['ds1', 'ds2', 'ds3']: + scenes[1][ds_id].attrs['start_time'] = datetime(2018, 1, 2) + scenes[1][ds_id].attrs['end_time'] = datetime(2018, 1, 2, 12) + if ds_id == 'ds3': + continue + scenes[0][ds_id].attrs['start_time'] = datetime(2018, 1, 1) + scenes[0][ds_id].attrs['end_time'] = datetime(2018, 1, 1, 12) + + mscn = MultiScene(scenes) + fn = str(tmp_path / + 'test_save_mp4_{name}_{start_time:%Y%m%d_%H}_{end_time:%Y%m%d_%H}.mp4') + writer_mock = mock.MagicMock() + with mock.patch('satpy.multiscene.imageio.get_writer') as get_writer: + get_writer.return_value = writer_mock + # force order of datasets by specifying them + mscn.save_animation(fn, datasets=['ds1', 'ds2', 'ds3'], client=False) + + # 2 saves for the first scene + 1 black frame + # 3 for the second scene + assert writer_mock.append_data.call_count == 3 + 3 + filenames = [os.path.basename(args[0][0]) for args in get_writer.call_args_list] + assert filenames[0] == 'test_save_mp4_ds1_20180101_00_20180102_12.mp4' + assert filenames[1] == 'test_save_mp4_ds2_20180101_00_20180102_12.mp4' + assert filenames[2] == 'test_save_mp4_ds3_20180102_00_20180102_12.mp4' + + # make sure that not specifying datasets still saves all of them + fn = str(tmp_path / + 'test_save_mp4_{name}_{start_time:%Y%m%d_%H}_{end_time:%Y%m%d_%H}.mp4') + writer_mock = mock.MagicMock() + with mock.patch('satpy.multiscene.imageio.get_writer') as get_writer: + get_writer.return_value = writer_mock + # force order of datasets by specifying them + mscn.save_animation(fn, client=False) + # the 'ds3' dataset isn't known to the first scene so it doesn't get saved + # 2 for first scene, 2 for second scene + assert writer_mock.append_data.call_count == 2 + 2 + assert "test_save_mp4_ds1_20180101_00_20180102_12.mp4" in filenames + assert "test_save_mp4_ds2_20180101_00_20180102_12.mp4" in filenames + assert "test_save_mp4_ds3_20180102_00_20180102_12.mp4" in filenames + + # test decorating and enhancing + + fn = str(tmp_path / + 'test-{name}_{start_time:%Y%m%d_%H}_{end_time:%Y%m%d_%H}-rich.mp4') + writer_mock = mock.MagicMock() + with mock.patch('satpy.multiscene.imageio.get_writer') as get_writer: + get_writer.return_value = writer_mock + mscn.save_animation( + fn, client=False, + enh_args={"decorate": { + "decorate": [{ + "text": { + "txt": + "Test {start_time:%Y-%m-%d %H:%M} - " + "{end_time:%Y-%m-%d %H:%M}"}}]}}) + assert writer_mock.append_data.call_count == 2 + 2 + assert ("2018-01-02" in smg.call_args_list[-1][1] + ["decorate"]["decorate"][0]["text"]["txt"]) diff --git a/satpy/tests/test_multiscene.py b/satpy/tests/test_multiscene.py deleted file mode 100644 index d969c0d101..0000000000 --- a/satpy/tests/test_multiscene.py +++ /dev/null @@ -1,866 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (c) 2018, 2022, 2023 Satpy developers -# -# This file is part of satpy. -# -# satpy is free software: you can redistribute it and/or modify it under the -# terms of the GNU General Public License as published by the Free Software -# Foundation, either version 3 of the License, or (at your option) any later -# version. -# -# satpy is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along with -# satpy. If not, see . -"""Unit tests for multiscene.py.""" - -import os -import shutil -import tempfile -import unittest -from datetime import datetime -from unittest import mock - -import dask.array as da -import pytest -import xarray as xr -from pyresample.geometry import AreaDefinition - -from satpy import DataQuery -from satpy.dataset.dataid import DataID, ModifierTuple, WavelengthRange - -# NOTE: -# The following fixtures are not defined in this file, but are used and injected by Pytest: -# - tmp_path - -DEFAULT_SHAPE = (5, 10) - -local_id_keys_config = {'name': { - 'required': True, -}, - 'wavelength': { - 'type': WavelengthRange, -}, - 'resolution': None, - 'calibration': { - 'enum': [ - 'reflectance', - 'brightness_temperature', - 'radiance', - 'counts' - ] -}, - 'polarization': None, - 'level': None, - 'modifiers': { - 'required': True, - 'default': ModifierTuple(), - 'type': ModifierTuple, -}, -} - - -def make_dataid(**items): - """Make a data id.""" - return DataID(local_id_keys_config, **items) - - -def _fake_get_enhanced_image(img, enhance=None, overlay=None, decorate=None): - from trollimage.xrimage import XRImage - return XRImage(img) - - -def _create_test_area(proj_str=None, shape=DEFAULT_SHAPE, extents=None): - """Create a test area definition.""" - from pyresample.geometry import AreaDefinition - from pyresample.utils import proj4_str_to_dict - if proj_str is None: - proj_str = '+proj=lcc +datum=WGS84 +ellps=WGS84 +lon_0=-95. ' \ - '+lat_0=25 +lat_1=25 +units=m +no_defs' - proj_dict = proj4_str_to_dict(proj_str) - extents = extents or (-1000., -1500., 1000., 1500.) - - return AreaDefinition( - 'test', - 'test', - 'test', - proj_dict, - shape[1], - shape[0], - extents - ) - - -def _create_test_int8_dataset(name, shape=DEFAULT_SHAPE, area=None, values=None): - """Create a test DataArray object.""" - import dask.array as da - import numpy as np - import xarray as xr - - return xr.DataArray( - da.ones(shape, dtype=np.uint8, chunks=shape) * values, dims=('y', 'x'), - attrs={'_FillValue': 255, - 'valid_range': [1, 15], - 'name': name, 'area': area, '_satpy_id_keys': local_id_keys_config}) - - -def _create_test_dataset(name, shape=DEFAULT_SHAPE, area=None, values=None): - """Create a test DataArray object.""" - import dask.array as da - import numpy as np - import xarray as xr - - if values: - return xr.DataArray( - da.ones(shape, dtype=np.float32, chunks=shape) * values, dims=('y', 'x'), - attrs={'name': name, 'area': area, '_satpy_id_keys': local_id_keys_config}) - - return xr.DataArray( - da.zeros(shape, dtype=np.float32, chunks=shape), dims=('y', 'x'), - attrs={'name': name, 'area': area, '_satpy_id_keys': local_id_keys_config}) - - -def _create_test_scenes(num_scenes=2, shape=DEFAULT_SHAPE, area=None): - """Create some test scenes for various test cases.""" - from satpy import Scene - ds1 = _create_test_dataset('ds1', shape=shape, area=area) - ds2 = _create_test_dataset('ds2', shape=shape, area=area) - scenes = [] - for _ in range(num_scenes): - scn = Scene() - scn['ds1'] = ds1.copy() - scn['ds2'] = ds2.copy() - scenes.append(scn) - return scenes - - -class TestMultiScene(unittest.TestCase): - """Test basic functionality of MultiScene.""" - - def test_init_empty(self): - """Test creating a multiscene with no children.""" - from satpy import MultiScene - MultiScene() - - def test_init_children(self): - """Test creating a multiscene with children.""" - from satpy import MultiScene - scenes = _create_test_scenes() - MultiScene(scenes) - - def test_properties(self): - """Test basic properties/attributes of the MultiScene.""" - from satpy import MultiScene - - area = _create_test_area() - scenes = _create_test_scenes(area=area) - ds1_id = make_dataid(name='ds1') - ds2_id = make_dataid(name='ds2') - ds3_id = make_dataid(name='ds3') - ds4_id = make_dataid(name='ds4') - - # Add a dataset to only one of the Scenes - scenes[1]['ds3'] = _create_test_dataset('ds3') - mscn = MultiScene(scenes) - - self.assertSetEqual(mscn.loaded_dataset_ids, - {ds1_id, ds2_id, ds3_id}) - self.assertSetEqual(mscn.shared_dataset_ids, {ds1_id, ds2_id}) - self.assertTrue(mscn.all_same_area) - - bigger_area = _create_test_area(shape=(20, 40)) - scenes[0]['ds4'] = _create_test_dataset('ds4', shape=(20, 40), - area=bigger_area) - - self.assertSetEqual(mscn.loaded_dataset_ids, - {ds1_id, ds2_id, ds3_id, ds4_id}) - self.assertSetEqual(mscn.shared_dataset_ids, {ds1_id, ds2_id}) - self.assertFalse(mscn.all_same_area) - - def test_from_files(self): - """Test creating a multiscene from multiple files.""" - from satpy import MultiScene - input_files_abi = [ - "OR_ABI-L1b-RadC-M3C01_G16_s20171171502203_e20171171504576_c20171171505018.nc", - "OR_ABI-L1b-RadC-M3C01_G16_s20171171507203_e20171171509576_c20171171510018.nc", - "OR_ABI-L1b-RadC-M3C01_G16_s20171171512203_e20171171514576_c20171171515017.nc", - "OR_ABI-L1b-RadC-M3C01_G16_s20171171517203_e20171171519577_c20171171520019.nc", - "OR_ABI-L1b-RadC-M3C01_G16_s20171171522203_e20171171524576_c20171171525020.nc", - "OR_ABI-L1b-RadC-M3C01_G16_s20171171527203_e20171171529576_c20171171530017.nc", - ] - input_files_glm = [ - "OR_GLM-L2-GLMC-M3_G16_s20171171500000_e20171171501000_c20380190314080.nc", - "OR_GLM-L2-GLMC-M3_G16_s20171171501000_e20171171502000_c20380190314080.nc", - "OR_GLM-L2-GLMC-M3_G16_s20171171502000_e20171171503000_c20380190314080.nc", - "OR_GLM-L2-GLMC-M3_G16_s20171171503000_e20171171504000_c20380190314080.nc", - "OR_GLM-L2-GLMC-M3_G16_s20171171504000_e20171171505000_c20380190314080.nc", - "OR_GLM-L2-GLMC-M3_G16_s20171171505000_e20171171506000_c20380190314080.nc", - "OR_GLM-L2-GLMC-M3_G16_s20171171506000_e20171171507000_c20380190314080.nc", - "OR_GLM-L2-GLMC-M3_G16_s20171171507000_e20171171508000_c20380190314080.nc", - ] - with mock.patch('satpy.multiscene.Scene') as scn_mock: - mscn = MultiScene.from_files( - input_files_abi, - reader='abi_l1b', - scene_kwargs={"reader_kwargs": {}}) - assert len(mscn.scenes) == 6 - calls = [mock.call( - filenames={'abi_l1b': [in_file_abi]}, - reader_kwargs={}) - for in_file_abi in input_files_abi] - scn_mock.assert_has_calls(calls) - - scn_mock.reset_mock() - with pytest.warns(DeprecationWarning): - mscn = MultiScene.from_files( - input_files_abi + input_files_glm, - reader=('abi_l1b', "glm_l2"), - group_keys=["start_time"], - ensure_all_readers=True, - time_threshold=30) - assert len(mscn.scenes) == 2 - calls = [mock.call( - filenames={'abi_l1b': [in_file_abi], 'glm_l2': [in_file_glm]}) - for (in_file_abi, in_file_glm) in - zip(input_files_abi[0:2], - [input_files_glm[2]] + [input_files_glm[7]])] - scn_mock.assert_has_calls(calls) - scn_mock.reset_mock() - mscn = MultiScene.from_files( - input_files_abi + input_files_glm, - reader=('abi_l1b', "glm_l2"), - group_keys=["start_time"], - ensure_all_readers=False, - time_threshold=30) - assert len(mscn.scenes) == 12 - - -class TestMultiSceneGrouping: - """Test dataset grouping in MultiScene.""" - - @pytest.fixture - def scene1(self): - """Create first test scene.""" - from satpy import Scene - scene = Scene() - dsid1 = make_dataid( - name="ds1", - resolution=123, - wavelength=(1, 2, 3), - polarization="H" - ) - scene[dsid1] = _create_test_dataset(name='ds1') - dsid2 = make_dataid( - name="ds2", - resolution=456, - wavelength=(4, 5, 6), - polarization="V" - ) - scene[dsid2] = _create_test_dataset(name='ds2') - return scene - - @pytest.fixture - def scene2(self): - """Create second test scene.""" - from satpy import Scene - scene = Scene() - dsid1 = make_dataid( - name="ds3", - resolution=123.1, - wavelength=(1.1, 2.1, 3.1), - polarization="H" - ) - scene[dsid1] = _create_test_dataset(name='ds3') - dsid2 = make_dataid( - name="ds4", - resolution=456.1, - wavelength=(4.1, 5.1, 6.1), - polarization="V" - ) - scene[dsid2] = _create_test_dataset(name='ds4') - return scene - - @pytest.fixture - def multi_scene(self, scene1, scene2): - """Create small multi scene for testing.""" - from satpy import MultiScene - return MultiScene([scene1, scene2]) - - @pytest.fixture - def groups(self): - """Get group definitions for the MultiScene.""" - return { - DataQuery(name='odd'): ['ds1', 'ds3'], - DataQuery(name='even'): ['ds2', 'ds4'] - } - - def test_multi_scene_grouping(self, multi_scene, groups, scene1): - """Test grouping a MultiScene.""" - multi_scene.group(groups) - shared_ids_exp = {make_dataid(name="odd"), make_dataid(name="even")} - assert multi_scene.shared_dataset_ids == shared_ids_exp - assert DataQuery(name='odd') not in scene1 - xr.testing.assert_allclose(multi_scene.scenes[0]["ds1"], scene1["ds1"]) - - def test_fails_to_add_multiple_datasets_from_the_same_scene_to_a_group(self, multi_scene): - """Test that multiple datasets from the same scene in one group fails.""" - groups = {DataQuery(name='mygroup'): ['ds1', 'ds2']} - multi_scene.group(groups) - with pytest.raises(ValueError): - next(multi_scene.scenes) - - -class TestMultiSceneSave(unittest.TestCase): - """Test saving a MultiScene to various formats.""" - - def setUp(self): - """Create temporary directory to save files to.""" - self.base_dir = tempfile.mkdtemp() - - def tearDown(self): - """Remove the temporary directory created for a test.""" - try: - shutil.rmtree(self.base_dir, ignore_errors=True) - except OSError: - pass - - @mock.patch('satpy.multiscene.get_enhanced_image', _fake_get_enhanced_image) - def test_save_mp4_distributed(self): - """Save a series of fake scenes to an mp4 video.""" - from satpy import MultiScene - area = _create_test_area() - scenes = _create_test_scenes(area=area) - - # Add a dataset to only one of the Scenes - scenes[1]['ds3'] = _create_test_dataset('ds3') - # Add a start and end time - for ds_id in ['ds1', 'ds2', 'ds3']: - scenes[1][ds_id].attrs['start_time'] = datetime(2018, 1, 2) - scenes[1][ds_id].attrs['end_time'] = datetime(2018, 1, 2, 12) - if ds_id == 'ds3': - continue - scenes[0][ds_id].attrs['start_time'] = datetime(2018, 1, 1) - scenes[0][ds_id].attrs['end_time'] = datetime(2018, 1, 1, 12) - - mscn = MultiScene(scenes) - fn = os.path.join( - self.base_dir, - 'test_save_mp4_{name}_{start_time:%Y%m%d_%H}_{end_time:%Y%m%d_%H}.mp4') - writer_mock = mock.MagicMock() - client_mock = mock.MagicMock() - client_mock.compute.side_effect = lambda x: tuple(v.compute() for v in x) - client_mock.gather.side_effect = lambda x: x - with mock.patch('satpy.multiscene.imageio.get_writer') as get_writer: - get_writer.return_value = writer_mock - # force order of datasets by specifying them - mscn.save_animation(fn, client=client_mock, datasets=['ds1', 'ds2', 'ds3']) - - # 2 saves for the first scene + 1 black frame - # 3 for the second scene - self.assertEqual(writer_mock.append_data.call_count, 3 + 3) - filenames = [os.path.basename(args[0][0]) for args in get_writer.call_args_list] - self.assertEqual(filenames[0], 'test_save_mp4_ds1_20180101_00_20180102_12.mp4') - self.assertEqual(filenames[1], 'test_save_mp4_ds2_20180101_00_20180102_12.mp4') - self.assertEqual(filenames[2], 'test_save_mp4_ds3_20180102_00_20180102_12.mp4') - - # Test no distributed client found - mscn = MultiScene(scenes) - fn = os.path.join( - self.base_dir, - 'test_save_mp4_{name}_{start_time:%Y%m%d_%H}_{end_time:%Y%m%d_%H}.mp4') - writer_mock = mock.MagicMock() - client_mock = mock.MagicMock() - client_mock.compute.side_effect = lambda x: tuple(v.compute() for v in x) - client_mock.gather.side_effect = lambda x: x - with mock.patch('satpy.multiscene.imageio.get_writer') as get_writer, \ - mock.patch('satpy.multiscene.get_client', mock.Mock(side_effect=ValueError("No client"))): - get_writer.return_value = writer_mock - # force order of datasets by specifying them - mscn.save_animation(fn, datasets=['ds1', 'ds2', 'ds3']) - - # 2 saves for the first scene + 1 black frame - # 3 for the second scene - self.assertEqual(writer_mock.append_data.call_count, 3 + 3) - filenames = [os.path.basename(args[0][0]) for args in get_writer.call_args_list] - self.assertEqual(filenames[0], 'test_save_mp4_ds1_20180101_00_20180102_12.mp4') - self.assertEqual(filenames[1], 'test_save_mp4_ds2_20180101_00_20180102_12.mp4') - self.assertEqual(filenames[2], 'test_save_mp4_ds3_20180102_00_20180102_12.mp4') - - @mock.patch('satpy.multiscene.get_enhanced_image', _fake_get_enhanced_image) - def test_save_mp4_no_distributed(self): - """Save a series of fake scenes to an mp4 video when distributed isn't available.""" - from satpy import MultiScene - area = _create_test_area() - scenes = _create_test_scenes(area=area) - - # Add a dataset to only one of the Scenes - scenes[1]['ds3'] = _create_test_dataset('ds3') - # Add a start and end time - for ds_id in ['ds1', 'ds2', 'ds3']: - scenes[1][ds_id].attrs['start_time'] = datetime(2018, 1, 2) - scenes[1][ds_id].attrs['end_time'] = datetime(2018, 1, 2, 12) - if ds_id == 'ds3': - continue - scenes[0][ds_id].attrs['start_time'] = datetime(2018, 1, 1) - scenes[0][ds_id].attrs['end_time'] = datetime(2018, 1, 1, 12) - - mscn = MultiScene(scenes) - fn = os.path.join( - self.base_dir, - 'test_save_mp4_{name}_{start_time:%Y%m%d_%H}_{end_time:%Y%m%d_%H}.mp4') - writer_mock = mock.MagicMock() - client_mock = mock.MagicMock() - client_mock.compute.side_effect = lambda x: tuple(v.compute() for v in x) - client_mock.gather.side_effect = lambda x: x - with mock.patch('satpy.multiscene.imageio.get_writer') as get_writer, \ - mock.patch('satpy.multiscene.get_client', None): - get_writer.return_value = writer_mock - # force order of datasets by specifying them - mscn.save_animation(fn, datasets=['ds1', 'ds2', 'ds3']) - - # 2 saves for the first scene + 1 black frame - # 3 for the second scene - self.assertEqual(writer_mock.append_data.call_count, 3 + 3) - filenames = [os.path.basename(args[0][0]) for args in get_writer.call_args_list] - self.assertEqual(filenames[0], 'test_save_mp4_ds1_20180101_00_20180102_12.mp4') - self.assertEqual(filenames[1], 'test_save_mp4_ds2_20180101_00_20180102_12.mp4') - self.assertEqual(filenames[2], 'test_save_mp4_ds3_20180102_00_20180102_12.mp4') - - @mock.patch('satpy.multiscene.get_enhanced_image', _fake_get_enhanced_image) - def test_save_datasets_simple(self): - """Save a series of fake scenes to an PNG images.""" - from satpy import MultiScene - area = _create_test_area() - scenes = _create_test_scenes(area=area) - - # Add a dataset to only one of the Scenes - scenes[1]['ds3'] = _create_test_dataset('ds3') - # Add a start and end time - for ds_id in ['ds1', 'ds2', 'ds3']: - scenes[1][ds_id].attrs['start_time'] = datetime(2018, 1, 2) - scenes[1][ds_id].attrs['end_time'] = datetime(2018, 1, 2, 12) - if ds_id == 'ds3': - continue - scenes[0][ds_id].attrs['start_time'] = datetime(2018, 1, 1) - scenes[0][ds_id].attrs['end_time'] = datetime(2018, 1, 1, 12) - - mscn = MultiScene(scenes) - client_mock = mock.MagicMock() - client_mock.compute.side_effect = lambda x: tuple(v for v in x) - client_mock.gather.side_effect = lambda x: x - with mock.patch('satpy.multiscene.Scene.save_datasets') as save_datasets: - save_datasets.return_value = [True] # some arbitrary return value - # force order of datasets by specifying them - mscn.save_datasets(base_dir=self.base_dir, client=False, datasets=['ds1', 'ds2', 'ds3'], - writer='simple_image') - - # 2 for each scene - self.assertEqual(save_datasets.call_count, 2) - - @mock.patch('satpy.multiscene.get_enhanced_image', _fake_get_enhanced_image) - def test_save_datasets_distributed_delayed(self): - """Test distributed save for writers returning delayed obejcts e.g. simple_image.""" - from dask.delayed import Delayed - - from satpy import MultiScene - area = _create_test_area() - scenes = _create_test_scenes(area=area) - - # Add a dataset to only one of the Scenes - scenes[1]['ds3'] = _create_test_dataset('ds3') - # Add a start and end time - for ds_id in ['ds1', 'ds2', 'ds3']: - scenes[1][ds_id].attrs['start_time'] = datetime(2018, 1, 2) - scenes[1][ds_id].attrs['end_time'] = datetime(2018, 1, 2, 12) - if ds_id == 'ds3': - continue - scenes[0][ds_id].attrs['start_time'] = datetime(2018, 1, 1) - scenes[0][ds_id].attrs['end_time'] = datetime(2018, 1, 1, 12) - - mscn = MultiScene(scenes) - client_mock = mock.MagicMock() - client_mock.compute.side_effect = lambda x: tuple(v for v in x) - client_mock.gather.side_effect = lambda x: x - future_mock = mock.MagicMock() - future_mock.__class__ = Delayed - with mock.patch('satpy.multiscene.Scene.save_datasets') as save_datasets: - save_datasets.return_value = [future_mock] # some arbitrary return value - # force order of datasets by specifying them - mscn.save_datasets(base_dir=self.base_dir, client=client_mock, datasets=['ds1', 'ds2', 'ds3'], - writer='simple_image') - - # 2 for each scene - self.assertEqual(save_datasets.call_count, 2) - - @mock.patch('satpy.multiscene.get_enhanced_image', _fake_get_enhanced_image) - def test_save_datasets_distributed_source_target(self): - """Test distributed save for writers returning sources and targets e.g. geotiff writer.""" - import dask.array as da - - from satpy import MultiScene - area = _create_test_area() - scenes = _create_test_scenes(area=area) - - # Add a dataset to only one of the Scenes - scenes[1]['ds3'] = _create_test_dataset('ds3') - # Add a start and end time - for ds_id in ['ds1', 'ds2', 'ds3']: - scenes[1][ds_id].attrs['start_time'] = datetime(2018, 1, 2) - scenes[1][ds_id].attrs['end_time'] = datetime(2018, 1, 2, 12) - if ds_id == 'ds3': - continue - scenes[0][ds_id].attrs['start_time'] = datetime(2018, 1, 1) - scenes[0][ds_id].attrs['end_time'] = datetime(2018, 1, 1, 12) - - mscn = MultiScene(scenes) - client_mock = mock.MagicMock() - client_mock.compute.side_effect = lambda x: tuple(v for v in x) - client_mock.gather.side_effect = lambda x: x - source_mock = mock.MagicMock() - source_mock.__class__ = da.Array - target_mock = mock.MagicMock() - with mock.patch('satpy.multiscene.Scene.save_datasets') as save_datasets: - save_datasets.return_value = [(source_mock, target_mock)] # some arbitrary return value - # force order of datasets by specifying them - with self.assertRaises(NotImplementedError): - mscn.save_datasets(base_dir=self.base_dir, client=client_mock, datasets=['ds1', 'ds2', 'ds3'], - writer='geotiff') - - def test_crop(self): - """Test the crop method.""" - import numpy as np - from pyresample.geometry import AreaDefinition - from xarray import DataArray - - from satpy import MultiScene, Scene - scene1 = Scene() - area_extent = (-5570248.477339745, -5561247.267842293, 5567248.074173927, - 5570248.477339745) - proj_dict = {'a': 6378169.0, 'b': 6356583.8, 'h': 35785831.0, - 'lon_0': 0.0, 'proj': 'geos', 'units': 'm'} - x_size = 3712 - y_size = 3712 - area_def = AreaDefinition( - 'test', 'test', 'test', - proj_dict, - x_size, - y_size, - area_extent, - ) - area_def2 = AreaDefinition( - 'test2', 'test2', 'test2', proj_dict, - x_size // 2, - y_size // 2, - area_extent, - ) - scene1["1"] = DataArray(np.zeros((y_size, x_size))) - scene1["2"] = DataArray(np.zeros((y_size, x_size)), dims=('y', 'x')) - scene1["3"] = DataArray(np.zeros((y_size, x_size)), dims=('y', 'x'), - attrs={'area': area_def}) - scene1["4"] = DataArray(np.zeros((y_size // 2, x_size // 2)), dims=('y', 'x'), - attrs={'area': area_def2}) - mscn = MultiScene([scene1]) - - # by lon/lat bbox - new_mscn = mscn.crop(ll_bbox=(-20., -5., 0, 0)) - new_scn1 = list(new_mscn.scenes)[0] - self.assertIn('1', new_scn1) - self.assertIn('2', new_scn1) - self.assertIn('3', new_scn1) - self.assertTupleEqual(new_scn1['1'].shape, (y_size, x_size)) - self.assertTupleEqual(new_scn1['2'].shape, (y_size, x_size)) - self.assertTupleEqual(new_scn1['3'].shape, (184, 714)) - self.assertTupleEqual(new_scn1['4'].shape, (92, 357)) - - -class TestBlendFuncs(): - """Test individual functions used for blending.""" - - def setup_method(self): - """Set up test functions.""" - self._line = 2 - self._column = 3 - - @pytest.fixture - def scene1_with_weights(self): - """Create first test scene with a dataset of weights.""" - from satpy import Scene - - area = _create_test_area() - scene = Scene() - dsid1 = make_dataid( - name="geo-ct", - resolution=3000, - modifiers=() - ) - scene[dsid1] = _create_test_int8_dataset(name='geo-ct', area=area, values=1) - wgt1 = _create_test_dataset(name='geo-ct-wgt', area=area, values=0) - - wgt1[self._line, :] = 2 - wgt1[:, self._column] = 2 - - dsid2 = make_dataid( - name="geo-cma", - resolution=3000, - modifiers=() - ) - scene[dsid2] = _create_test_int8_dataset(name='geo-cma', area=area, values=2) - wgt2 = _create_test_dataset(name='geo-cma-wgt', area=area, values=0) - - return scene, [wgt1, wgt2] - - @pytest.fixture - def scene2_with_weights(self): - """Create second test scene.""" - from satpy import Scene - - area = _create_test_area() - scene = Scene() - dsid1 = make_dataid( - name="polar-ct", - modifiers=() - ) - scene[dsid1] = _create_test_int8_dataset(name='polar-ct', area=area, values=3) - scene[dsid1][-1, :] = scene[dsid1].attrs['_FillValue'] - wgt1 = _create_test_dataset(name='polar-ct-wgt', area=area, values=1) - - dsid2 = make_dataid( - name="polar-cma", - modifiers=() - ) - scene[dsid2] = _create_test_int8_dataset(name='polar-cma', area=area, values=4) - wgt2 = _create_test_dataset(name='polar-cma-wgt', area=area, values=1) - return scene, [wgt1, wgt2] - - @pytest.fixture - def multi_scene_and_weights(self, scene1_with_weights, scene2_with_weights): - """Create small multi-scene for testing.""" - from satpy import MultiScene - scene1, weights1 = scene1_with_weights - scene2, weights2 = scene2_with_weights - - return MultiScene([scene1, scene2]), [weights1, weights2] - - @pytest.fixture - def groups(self): - """Get group definitions for the MultiScene.""" - return { - DataQuery(name='CloudType'): ['geo-ct', 'polar-ct'], - DataQuery(name='CloudMask'): ['geo-cma', 'polar-cma'] - } - - def test_blend_two_scenes_using_stack(self, multi_scene_and_weights, groups, - scene1_with_weights, scene2_with_weights): - """Test blending two scenes by stacking them on top of each other using function 'stack'.""" - from satpy.multiscene import stack - - multi_scene, weights = multi_scene_and_weights - scene1, weights1 = scene1_with_weights - scene2, weights2 = scene2_with_weights - - multi_scene.group(groups) - - resampled = multi_scene - stacked = resampled.blend(blend_function=stack) - - expected = scene2['polar-ct'].copy() - expected[-1, :] = scene1['geo-ct'][-1, :] - - xr.testing.assert_equal(stacked['CloudType'].compute(), expected.compute()) - - def test_blend_two_scenes_using_stack_weighted(self, multi_scene_and_weights, groups, - scene1_with_weights, scene2_with_weights): - """Test stacking two scenes using weights.""" - from functools import partial - - from satpy.multiscene import stack - - multi_scene, weights = multi_scene_and_weights - scene1, weights1 = scene1_with_weights - scene2, weights2 = scene2_with_weights - - simple_groups = {DataQuery(name='CloudType'): groups[DataQuery(name='CloudType')]} - multi_scene.group(simple_groups) - - weights = [weights[0][0], weights[1][0]] - stack_with_weights = partial(stack, weights=weights) - weighted_blend = multi_scene.blend(blend_function=stack_with_weights) - - expected = scene2['polar-ct'] - expected[self._line, :] = scene1['geo-ct'][self._line, :] - expected[:, self._column] = scene1['geo-ct'][:, self._column] - expected[-1, :] = scene1['geo-ct'][-1, :] - - result = weighted_blend['CloudType'].compute() - xr.testing.assert_equal(result, expected.compute()) - - @pytest.fixture - def datasets_and_weights(self): - """X-Array datasets with area definition plus weights for input to tests.""" - shape = (8, 12) - area = AreaDefinition('test', 'test', 'test', - {'proj': 'geos', 'lon_0': -95.5, 'h': 35786023.0}, - shape[1], shape[0], [-200, -200, 200, 200]) - - ds1 = xr.DataArray(da.ones(shape, chunks=-1), dims=('y', 'x'), - attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) - ds2 = xr.DataArray(da.ones(shape, chunks=-1) * 2, dims=('y', 'x'), - attrs={'start_time': datetime(2018, 1, 1, 1, 0, 0), 'area': area}) - ds3 = xr.DataArray(da.ones(shape, chunks=-1) * 3, dims=('y', 'x'), - attrs={'start_time': datetime(2018, 1, 1, 1, 0, 0), 'area': area}) - - ds4 = xr.DataArray(da.zeros(shape, chunks=-1), dims=('y', 'time'), - attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) - ds5 = xr.DataArray(da.zeros(shape, chunks=-1), dims=('y', 'time'), - attrs={'start_time': datetime(2018, 1, 1, 1, 0, 0), 'area': area}) - - wgt1 = xr.DataArray(da.ones(shape, chunks=-1), dims=('y', 'x'), - attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) - wgt2 = xr.DataArray(da.zeros(shape, chunks=-1), dims=('y', 'x'), - attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) - wgt3 = xr.DataArray(da.zeros(shape, chunks=-1), dims=('y', 'x'), - attrs={'start_time': datetime(2018, 1, 1, 0, 0, 0), 'area': area}) - - datastruct = {'shape': shape, - 'area': area, - 'datasets': [ds1, ds2, ds3, ds4, ds5], - 'weights': [wgt1, wgt2, wgt3]} - return datastruct - - @pytest.mark.parametrize(('line', 'column',), - [(2, 3), (4, 5)] - ) - def test_blend_function_stack_weighted(self, datasets_and_weights, line, column): - """Test the 'stack_weighted' function.""" - from functools import partial - - from satpy.multiscene import stack - - input_data = datasets_and_weights - - input_data['weights'][1][line, :] = 2 - input_data['weights'][2][:, column] = 2 - - stack_with_weights = partial(stack, weights=input_data['weights']) - blend_result = stack_with_weights(input_data['datasets'][0:3]) - - ds1 = input_data['datasets'][0] - ds2 = input_data['datasets'][1] - ds3 = input_data['datasets'][2] - expected = ds1.copy() - expected[:, column] = ds3[:, column] - expected[line, :] = ds2[line, :] - - xr.testing.assert_equal(blend_result.compute(), expected.compute()) - assert expected.attrs == blend_result.attrs - - def test_blend_function_stack(self, datasets_and_weights): - """Test the 'stack' function.""" - from satpy.multiscene import stack - - input_data = datasets_and_weights - - ds1 = input_data['datasets'][0] - ds2 = input_data['datasets'][1] - - res = stack([ds1, ds2]) - expected = ds2.copy() - - xr.testing.assert_equal(res.compute(), expected.compute()) - # assert expected.attrs == res.attrs - # FIXME! Looks like the attributes are taken from the first dataset. Should - # be like that? So in this case the datetime is different from "expected" - # (= in this case the last dataset in the stack, the one on top) - - def test_timeseries(self, datasets_and_weights): - """Test the 'timeseries' function.""" - from satpy.multiscene import timeseries - - input_data = datasets_and_weights - - ds1 = input_data['datasets'][0] - ds2 = input_data['datasets'][1] - ds4 = input_data['datasets'][2] - ds4 = input_data['datasets'][3] - ds5 = input_data['datasets'][4] - - res = timeseries([ds1, ds2]) - res2 = timeseries([ds4, ds5]) - assert isinstance(res, xr.DataArray) - assert isinstance(res2, xr.DataArray) - assert (2, ds1.shape[0], ds1.shape[1]) == res.shape - assert (ds4.shape[0], ds4.shape[1]+ds5.shape[1]) == res2.shape - - -@mock.patch('satpy.multiscene.get_enhanced_image') -def test_save_mp4(smg, tmp_path): - """Save a series of fake scenes to an mp4 video.""" - from satpy import MultiScene - area = _create_test_area() - scenes = _create_test_scenes(area=area) - smg.side_effect = _fake_get_enhanced_image - - # Add a dataset to only one of the Scenes - scenes[1]['ds3'] = _create_test_dataset('ds3') - # Add a start and end time - for ds_id in ['ds1', 'ds2', 'ds3']: - scenes[1][ds_id].attrs['start_time'] = datetime(2018, 1, 2) - scenes[1][ds_id].attrs['end_time'] = datetime(2018, 1, 2, 12) - if ds_id == 'ds3': - continue - scenes[0][ds_id].attrs['start_time'] = datetime(2018, 1, 1) - scenes[0][ds_id].attrs['end_time'] = datetime(2018, 1, 1, 12) - - mscn = MultiScene(scenes) - fn = str(tmp_path / - 'test_save_mp4_{name}_{start_time:%Y%m%d_%H}_{end_time:%Y%m%d_%H}.mp4') - writer_mock = mock.MagicMock() - with mock.patch('satpy.multiscene.imageio.get_writer') as get_writer: - get_writer.return_value = writer_mock - # force order of datasets by specifying them - mscn.save_animation(fn, datasets=['ds1', 'ds2', 'ds3'], client=False) - - # 2 saves for the first scene + 1 black frame - # 3 for the second scene - assert writer_mock.append_data.call_count == 3 + 3 - filenames = [os.path.basename(args[0][0]) for args in get_writer.call_args_list] - assert filenames[0] == 'test_save_mp4_ds1_20180101_00_20180102_12.mp4' - assert filenames[1] == 'test_save_mp4_ds2_20180101_00_20180102_12.mp4' - assert filenames[2] == 'test_save_mp4_ds3_20180102_00_20180102_12.mp4' - - # make sure that not specifying datasets still saves all of them - fn = str(tmp_path / - 'test_save_mp4_{name}_{start_time:%Y%m%d_%H}_{end_time:%Y%m%d_%H}.mp4') - writer_mock = mock.MagicMock() - with mock.patch('satpy.multiscene.imageio.get_writer') as get_writer: - get_writer.return_value = writer_mock - # force order of datasets by specifying them - mscn.save_animation(fn, client=False) - # the 'ds3' dataset isn't known to the first scene so it doesn't get saved - # 2 for first scene, 2 for second scene - assert writer_mock.append_data.call_count == 2 + 2 - assert "test_save_mp4_ds1_20180101_00_20180102_12.mp4" in filenames - assert "test_save_mp4_ds2_20180101_00_20180102_12.mp4" in filenames - assert "test_save_mp4_ds3_20180102_00_20180102_12.mp4" in filenames - - # test decorating and enhancing - - fn = str(tmp_path / - 'test-{name}_{start_time:%Y%m%d_%H}_{end_time:%Y%m%d_%H}-rich.mp4') - writer_mock = mock.MagicMock() - with mock.patch('satpy.multiscene.imageio.get_writer') as get_writer: - get_writer.return_value = writer_mock - mscn.save_animation( - fn, client=False, - enh_args={"decorate": { - "decorate": [{ - "text": { - "txt": - "Test {start_time:%Y-%m-%d %H:%M} - " - "{end_time:%Y-%m-%d %H:%M}"}}]}}) - assert writer_mock.append_data.call_count == 2 + 2 - assert ("2018-01-02" in smg.call_args_list[-1][1] - ["decorate"]["decorate"][0]["text"]["txt"]) From b28df3f8e461bbbeb6b8691fe52929292d38f5b8 Mon Sep 17 00:00:00 2001 From: Adam Dybbroe Date: Tue, 24 Jan 2023 04:33:41 +0100 Subject: [PATCH 09/30] Update satpy/modifiers/angles.py Co-authored-by: David Hoese --- satpy/modifiers/angles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/modifiers/angles.py b/satpy/modifiers/angles.py index 9b0f9ea1cf..878c33e908 100644 --- a/satpy/modifiers/angles.py +++ b/satpy/modifiers/angles.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2021, 2023 Satpy developers +# Copyright (c) 2021-2023 Satpy developers # # This file is part of satpy. # From e262e12cefca07f85cf2d3136ef7552e512ae071 Mon Sep 17 00:00:00 2001 From: Adam Dybbroe Date: Tue, 24 Jan 2023 04:36:32 +0100 Subject: [PATCH 10/30] Update satpy/multiscene.py Co-authored-by: David Hoese --- satpy/multiscene.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/multiscene.py b/satpy/multiscene.py index 37906e53a5..40f3372f1d 100644 --- a/satpy/multiscene.py +++ b/satpy/multiscene.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2016-2019, 2022, 2023 Satpy developers +# Copyright (c) 2016-2023 Satpy developers # # This file is part of satpy. # From 24b915351e423509aa1d7fc755fadd040284072b Mon Sep 17 00:00:00 2001 From: Adam Dybbroe Date: Tue, 24 Jan 2023 04:38:53 +0100 Subject: [PATCH 11/30] Improve doc string in satpy/multiscene.py Co-authored-by: David Hoese --- satpy/multiscene.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/multiscene.py b/satpy/multiscene.py index 40f3372f1d..38152a57d0 100644 --- a/satpy/multiscene.py +++ b/satpy/multiscene.py @@ -46,7 +46,7 @@ def stack(datasets, weights=None): - """Combine a series of datasets together. + """Overlay a series of datasets together. On default, datasets are stacked on top of each other, so the last one is on top. But if a sequence of weights arrays are provided the datasets will From 695b98dfabf045c8acd571f3a6da9ca34dbd055b Mon Sep 17 00:00:00 2001 From: Adam Dybbroe Date: Tue, 24 Jan 2023 04:40:18 +0100 Subject: [PATCH 12/30] Clarify doc string in satpy/multiscene.py Co-authored-by: David Hoese --- satpy/multiscene.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/multiscene.py b/satpy/multiscene.py index 38152a57d0..e5270e8111 100644 --- a/satpy/multiscene.py +++ b/satpy/multiscene.py @@ -48,8 +48,8 @@ def stack(datasets, weights=None): """Overlay a series of datasets together. - On default, datasets are stacked on top of each other, so the last one is - on top. But if a sequence of weights arrays are provided the datasets will + By default, datasets are stacked on top of each other, so the last one applied is + on top. If a sequence of weights arrays are provided the datasets will be combined according to those weights. The result will be a composite dataset where the data in each pixel is coming from the dataset having the highest weight. From 6627d48d34d4cfddfc736a2bb4bee76c8a7d4cde Mon Sep 17 00:00:00 2001 From: Adam Dybbroe Date: Tue, 24 Jan 2023 04:40:45 +0100 Subject: [PATCH 13/30] Update satpy/tests/multiscene_tests/test_save_animation.py Co-authored-by: David Hoese --- satpy/tests/multiscene_tests/test_save_animation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/satpy/tests/multiscene_tests/test_save_animation.py b/satpy/tests/multiscene_tests/test_save_animation.py index 6b62033059..42b89d930f 100644 --- a/satpy/tests/multiscene_tests/test_save_animation.py +++ b/satpy/tests/multiscene_tests/test_save_animation.py @@ -16,7 +16,6 @@ # # You should have received a copy of the GNU General Public License along with # satpy. If not, see . - """Unit tests for saving animations using Multiscene.""" import os From 80ff2a01c8a58bd72da4ab304e98d55a185f7b29 Mon Sep 17 00:00:00 2001 From: Adam Dybbroe Date: Tue, 24 Jan 2023 04:42:18 +0100 Subject: [PATCH 14/30] Make sure module headers are consistent Co-authored-by: David Hoese --- satpy/tests/multiscene_tests/test_save_animation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/satpy/tests/multiscene_tests/test_save_animation.py b/satpy/tests/multiscene_tests/test_save_animation.py index 42b89d930f..9e2d7f24ad 100644 --- a/satpy/tests/multiscene_tests/test_save_animation.py +++ b/satpy/tests/multiscene_tests/test_save_animation.py @@ -1,6 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- - # Copyright (c) 2018-2023 Satpy developers # # This file is part of satpy. From 818b0f1b94d748c5e90299bb0c91d81fc3beba69 Mon Sep 17 00:00:00 2001 From: Adam Dybbroe Date: Tue, 24 Jan 2023 04:42:49 +0100 Subject: [PATCH 15/30] Update satpy/tests/multiscene_tests/test_blend.py Co-authored-by: David Hoese --- satpy/tests/multiscene_tests/test_blend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/tests/multiscene_tests/test_blend.py b/satpy/tests/multiscene_tests/test_blend.py index bf3e96d711..65e038ab76 100644 --- a/satpy/tests/multiscene_tests/test_blend.py +++ b/satpy/tests/multiscene_tests/test_blend.py @@ -30,7 +30,7 @@ from satpy.tests.multiscene_tests import _create_test_area, _create_test_dataset, _create_test_int8_dataset, make_dataid -class TestBlendFuncs(): +class TestBlendFuncs: """Test individual functions used for blending.""" def setup_method(self): From c2a90d85326c88e970e51f161ce58ef6871aa482 Mon Sep 17 00:00:00 2001 From: Adam Dybbroe Date: Tue, 24 Jan 2023 04:43:29 +0100 Subject: [PATCH 16/30] Update copyright in module header Co-authored-by: David Hoese --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6c6c51270a..e2b33ced9e 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2009-2022 Satpy developers +# Copyright (c) 2009-2023 Satpy developers # # This file is part of satpy. # From 248d449f07e8a22dac7d08d8fdeeec24edaf984e Mon Sep 17 00:00:00 2001 From: Adam Dybbroe Date: Tue, 24 Jan 2023 04:52:21 +0100 Subject: [PATCH 17/30] Update satpy/multiscene.py Co-authored-by: David Hoese --- satpy/multiscene.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/multiscene.py b/satpy/multiscene.py index e5270e8111..0c776a8a77 100644 --- a/satpy/multiscene.py +++ b/satpy/multiscene.py @@ -61,8 +61,8 @@ def stack(datasets, weights=None): base = datasets[0].copy() for dataset in datasets[1:]: try: - base = base.where(dataset == dataset._FillValue, dataset) - except AttributeError: + base = base.where(dataset == dataset.attrs["_FillValue"], dataset) + except KeyError: base = base.where(dataset.isnull(), dataset) return base From f4b0874b002caa4d95efbfe7e191932a72045fb9 Mon Sep 17 00:00:00 2001 From: Adam Dybbroe Date: Tue, 24 Jan 2023 04:53:36 +0100 Subject: [PATCH 18/30] Use attrs keys to access the fill value Co-authored-by: David Hoese --- satpy/multiscene.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/multiscene.py b/satpy/multiscene.py index 0c776a8a77..af3d4809a2 100644 --- a/satpy/multiscene.py +++ b/satpy/multiscene.py @@ -72,8 +72,8 @@ def _stack_weighted(datasets, weights): # Go through weights and set to zero where corresponding datasets have a value equals _FillValue or nan for i, dataset in enumerate(datasets): try: - weights[i] = xr.where(dataset == dataset._FillValue, 0, weights[i]) - except AttributeError: + weights[i] = xr.where(dataset == dataset.attrs["_FillValue"], 0, weights[i]) + except KeyError: weights[i] = xr.where(dataset.isnull(), 0, weights[i]) indices = da.argmax(da.dstack(weights), axis=-1) From 0def84aff535d232cfc4cbd5516c3fb6b7c39e77 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Tue, 24 Jan 2023 05:16:03 +0100 Subject: [PATCH 19/30] Remove dupplicate code, and use make_dataid from utils instead Signed-off-by: Adam.Dybbroe --- satpy/tests/multiscene_tests/__init__.py | 7 +------ satpy/tests/multiscene_tests/test_blend.py | 3 ++- satpy/tests/multiscene_tests/test_misc.py | 3 ++- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/satpy/tests/multiscene_tests/__init__.py b/satpy/tests/multiscene_tests/__init__.py index f2a27534be..a788493181 100644 --- a/satpy/tests/multiscene_tests/__init__.py +++ b/satpy/tests/multiscene_tests/__init__.py @@ -26,7 +26,7 @@ import xarray as xr from pyresample.geometry import AreaDefinition -from satpy.dataset.dataid import DataID, ModifierTuple, WavelengthRange +from satpy.dataset.dataid import ModifierTuple, WavelengthRange DEFAULT_SHAPE = (5, 10) @@ -55,11 +55,6 @@ } -def make_dataid(**items): - """Make a data id.""" - return DataID(local_id_keys_config, **items) - - def _fake_get_enhanced_image(img, enhance=None, overlay=None, decorate=None): from trollimage.xrimage import XRImage return XRImage(img) diff --git a/satpy/tests/multiscene_tests/test_blend.py b/satpy/tests/multiscene_tests/test_blend.py index bf3e96d711..4373bba9a0 100644 --- a/satpy/tests/multiscene_tests/test_blend.py +++ b/satpy/tests/multiscene_tests/test_blend.py @@ -27,7 +27,8 @@ from pyresample.geometry import AreaDefinition from satpy import DataQuery -from satpy.tests.multiscene_tests import _create_test_area, _create_test_dataset, _create_test_int8_dataset, make_dataid +from satpy.tests.multiscene_tests import _create_test_area, _create_test_dataset, _create_test_int8_dataset +from satpy.tests.utils import make_dataid class TestBlendFuncs(): diff --git a/satpy/tests/multiscene_tests/test_misc.py b/satpy/tests/multiscene_tests/test_misc.py index 62340ff0ad..d6116a0efe 100644 --- a/satpy/tests/multiscene_tests/test_misc.py +++ b/satpy/tests/multiscene_tests/test_misc.py @@ -25,7 +25,8 @@ import xarray as xr from satpy import DataQuery -from satpy.tests.multiscene_tests import _create_test_area, _create_test_dataset, _create_test_scenes, make_dataid +from satpy.tests.multiscene_tests import _create_test_area, _create_test_dataset, _create_test_scenes +from satpy.tests.utils import make_dataid class TestMultiScene(unittest.TestCase): From 505aa24eb02ea5f9ce38810a04d56a2b4d5bffa5 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Tue, 24 Jan 2023 11:41:04 +0100 Subject: [PATCH 20/30] Moving test utilities from __init__ to separate module Signed-off-by: Adam.Dybbroe --- satpy/tests/multiscene_tests/__init__.py | 97 --------------- satpy/tests/multiscene_tests/test_blend.py | 2 +- satpy/tests/multiscene_tests/test_misc.py | 2 +- .../multiscene_tests/test_save_animation.py | 6 +- satpy/tests/multiscene_tests/test_utils.py | 115 ++++++++++++++++++ 5 files changed, 122 insertions(+), 100 deletions(-) create mode 100644 satpy/tests/multiscene_tests/test_utils.py diff --git a/satpy/tests/multiscene_tests/__init__.py b/satpy/tests/multiscene_tests/__init__.py index a788493181..0b191d1436 100644 --- a/satpy/tests/multiscene_tests/__init__.py +++ b/satpy/tests/multiscene_tests/__init__.py @@ -16,100 +16,3 @@ # You should have received a copy of the GNU General Public License along with # satpy. If not, see . """Unit tests for Multiscene.""" - -# NOTE: -# The following fixtures are not defined in this file, but are used and injected by Pytest: -# - tmp_path - -import dask.array as da -import numpy as np -import xarray as xr -from pyresample.geometry import AreaDefinition - -from satpy.dataset.dataid import ModifierTuple, WavelengthRange - -DEFAULT_SHAPE = (5, 10) - -local_id_keys_config = {'name': { - 'required': True, -}, - 'wavelength': { - 'type': WavelengthRange, -}, - 'resolution': None, - 'calibration': { - 'enum': [ - 'reflectance', - 'brightness_temperature', - 'radiance', - 'counts' - ] -}, - 'polarization': None, - 'level': None, - 'modifiers': { - 'required': True, - 'default': ModifierTuple(), - 'type': ModifierTuple, -}, -} - - -def _fake_get_enhanced_image(img, enhance=None, overlay=None, decorate=None): - from trollimage.xrimage import XRImage - return XRImage(img) - - -def _create_test_area(proj_str=None, shape=DEFAULT_SHAPE, extents=None): - """Create a test area definition.""" - from pyresample.utils import proj4_str_to_dict - if proj_str is None: - proj_str = '+proj=lcc +datum=WGS84 +ellps=WGS84 +lon_0=-95. ' \ - '+lat_0=25 +lat_1=25 +units=m +no_defs' - proj_dict = proj4_str_to_dict(proj_str) - extents = extents or (-1000., -1500., 1000., 1500.) - - return AreaDefinition( - 'test', - 'test', - 'test', - proj_dict, - shape[1], - shape[0], - extents - ) - - -def _create_test_int8_dataset(name, shape=DEFAULT_SHAPE, area=None, values=None): - """Create a test DataArray object.""" - return xr.DataArray( - da.ones(shape, dtype=np.uint8, chunks=shape) * values, dims=('y', 'x'), - attrs={'_FillValue': 255, - 'valid_range': [1, 15], - 'name': name, 'area': area, '_satpy_id_keys': local_id_keys_config}) - - -def _create_test_dataset(name, shape=DEFAULT_SHAPE, area=None, values=None): - """Create a test DataArray object.""" - if values: - return xr.DataArray( - da.ones(shape, dtype=np.float32, chunks=shape) * values, dims=('y', 'x'), - attrs={'name': name, 'area': area, '_satpy_id_keys': local_id_keys_config}) - - return xr.DataArray( - da.zeros(shape, dtype=np.float32, chunks=shape), dims=('y', 'x'), - attrs={'name': name, 'area': area, '_satpy_id_keys': local_id_keys_config}) - - -def _create_test_scenes(num_scenes=2, shape=DEFAULT_SHAPE, area=None): - """Create some test scenes for various test cases.""" - from satpy import Scene - ds1 = _create_test_dataset('ds1', shape=shape, area=area) - ds2 = _create_test_dataset('ds2', shape=shape, area=area) - scenes = [] - for _ in range(num_scenes): - scn = Scene() - scn['ds1'] = ds1.copy() - scn['ds2'] = ds2.copy() - scenes.append(scn) - return scenes diff --git a/satpy/tests/multiscene_tests/test_blend.py b/satpy/tests/multiscene_tests/test_blend.py index 70e623a233..8153067bc0 100644 --- a/satpy/tests/multiscene_tests/test_blend.py +++ b/satpy/tests/multiscene_tests/test_blend.py @@ -27,7 +27,7 @@ from pyresample.geometry import AreaDefinition from satpy import DataQuery -from satpy.tests.multiscene_tests import _create_test_area, _create_test_dataset, _create_test_int8_dataset +from satpy.tests.multiscene_tests.test_utils import _create_test_area, _create_test_dataset, _create_test_int8_dataset from satpy.tests.utils import make_dataid diff --git a/satpy/tests/multiscene_tests/test_misc.py b/satpy/tests/multiscene_tests/test_misc.py index d6116a0efe..5f8ea4dff9 100644 --- a/satpy/tests/multiscene_tests/test_misc.py +++ b/satpy/tests/multiscene_tests/test_misc.py @@ -25,7 +25,7 @@ import xarray as xr from satpy import DataQuery -from satpy.tests.multiscene_tests import _create_test_area, _create_test_dataset, _create_test_scenes +from satpy.tests.multiscene_tests.test_utils import _create_test_area, _create_test_dataset, _create_test_scenes from satpy.tests.utils import make_dataid diff --git a/satpy/tests/multiscene_tests/test_save_animation.py b/satpy/tests/multiscene_tests/test_save_animation.py index 9e2d7f24ad..e5f29ad573 100644 --- a/satpy/tests/multiscene_tests/test_save_animation.py +++ b/satpy/tests/multiscene_tests/test_save_animation.py @@ -17,6 +17,10 @@ # satpy. If not, see . """Unit tests for saving animations using Multiscene.""" +# NOTE: +# The following fixtures are not defined in this file, but are used and injected by Pytest: +# - tmp_path + import os import shutil import tempfile @@ -24,7 +28,7 @@ from datetime import datetime from unittest import mock -from satpy.tests.multiscene_tests import ( +from satpy.tests.multiscene_tests.test_utils import ( _create_test_area, _create_test_dataset, _create_test_scenes, diff --git a/satpy/tests/multiscene_tests/test_utils.py b/satpy/tests/multiscene_tests/test_utils.py new file mode 100644 index 0000000000..963f4b8f85 --- /dev/null +++ b/satpy/tests/multiscene_tests/test_utils.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2018-2023 Satpy developers +# +# This file is part of satpy. +# +# satpy is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# satpy is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# satpy. If not, see . +"""Utilties to assist testing the Multiscene functionality. + +Creating fake test data for use in the other Multiscene test modules. +""" + + +import dask.array as da +import numpy as np +import xarray as xr +from pyresample.geometry import AreaDefinition + +from satpy.dataset.dataid import ModifierTuple, WavelengthRange + +DEFAULT_SHAPE = (5, 10) + +local_id_keys_config = {'name': { + 'required': True, +}, + 'wavelength': { + 'type': WavelengthRange, +}, + 'resolution': None, + 'calibration': { + 'enum': [ + 'reflectance', + 'brightness_temperature', + 'radiance', + 'counts' + ] +}, + 'polarization': None, + 'level': None, + 'modifiers': { + 'required': True, + 'default': ModifierTuple(), + 'type': ModifierTuple, +}, +} + + +def _fake_get_enhanced_image(img, enhance=None, overlay=None, decorate=None): + from trollimage.xrimage import XRImage + return XRImage(img) + + +def _create_test_area(proj_str=None, shape=DEFAULT_SHAPE, extents=None): + """Create a test area definition.""" + from pyresample.utils import proj4_str_to_dict + if proj_str is None: + proj_str = '+proj=lcc +datum=WGS84 +ellps=WGS84 +lon_0=-95. ' \ + '+lat_0=25 +lat_1=25 +units=m +no_defs' + proj_dict = proj4_str_to_dict(proj_str) + extents = extents or (-1000., -1500., 1000., 1500.) + + return AreaDefinition( + 'test', + 'test', + 'test', + proj_dict, + shape[1], + shape[0], + extents + ) + + +def _create_test_int8_dataset(name, shape=DEFAULT_SHAPE, area=None, values=None): + """Create a test DataArray object.""" + return xr.DataArray( + da.ones(shape, dtype=np.uint8, chunks=shape) * values, dims=('y', 'x'), + attrs={'_FillValue': 255, + 'valid_range': [1, 15], + 'name': name, 'area': area, '_satpy_id_keys': local_id_keys_config}) + + +def _create_test_dataset(name, shape=DEFAULT_SHAPE, area=None, values=None): + """Create a test DataArray object.""" + if values: + return xr.DataArray( + da.ones(shape, dtype=np.float32, chunks=shape) * values, dims=('y', 'x'), + attrs={'name': name, 'area': area, '_satpy_id_keys': local_id_keys_config}) + + return xr.DataArray( + da.zeros(shape, dtype=np.float32, chunks=shape), dims=('y', 'x'), + attrs={'name': name, 'area': area, '_satpy_id_keys': local_id_keys_config}) + + +def _create_test_scenes(num_scenes=2, shape=DEFAULT_SHAPE, area=None): + """Create some test scenes for various test cases.""" + from satpy import Scene + ds1 = _create_test_dataset('ds1', shape=shape, area=area) + ds2 = _create_test_dataset('ds2', shape=shape, area=area) + scenes = [] + for _ in range(num_scenes): + scn = Scene() + scn['ds1'] = ds1.copy() + scn['ds2'] = ds2.copy() + scenes.append(scn) + return scenes From b5adf6a50719767e9a9257435973ae761a3e67d8 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Thu, 26 Jan 2023 14:50:53 +0100 Subject: [PATCH 21/30] Minor refactoring of the blend/stack functions and add combining of times for the stack-weighted multiscene Signed-off-by: Adam.Dybbroe --- satpy/multiscene.py | 46 +++++++-- satpy/tests/multiscene_tests/test_blend.py | 103 ++++++++++++++++++--- 2 files changed, 128 insertions(+), 21 deletions(-) diff --git a/satpy/multiscene.py b/satpy/multiscene.py index af3d4809a2..22d566d1ff 100644 --- a/satpy/multiscene.py +++ b/satpy/multiscene.py @@ -20,6 +20,7 @@ import copy import logging import warnings +from datetime import datetime from queue import Queue from threading import Thread @@ -45,7 +46,7 @@ log = logging.getLogger(__name__) -def stack(datasets, weights=None): +def stack(datasets, weights=None, combine_times=True): """Overlay a series of datasets together. By default, datasets are stacked on top of each other, so the last one applied is @@ -56,7 +57,7 @@ def stack(datasets, weights=None): """ if weights: - return _stack_weighted(datasets, weights) + return _stack_weighted(datasets, weights, combine_times) base = datasets[0].copy() for dataset in datasets[1:]: @@ -64,24 +65,49 @@ def stack(datasets, weights=None): base = base.where(dataset == dataset.attrs["_FillValue"], dataset) except KeyError: base = base.where(dataset.isnull(), dataset) + return base -def _stack_weighted(datasets, weights): +def _stack_weighted(datasets, weights, combine_times): """Stack datasets using weights.""" - # Go through weights and set to zero where corresponding datasets have a value equals _FillValue or nan + weights = set_weights_to_zero_where_invalid(datasets, weights) + + indices = da.argmax(da.dstack(weights), axis=-1) + attrs = combine_metadata(*[x.attrs for x in datasets]) + + if combine_times: + attrs['start_time'], attrs['end_time'] = get_combined_start_end_times(*[x.attrs for x in datasets]) + + dims = datasets[0].dims + weighted_array = xr.DataArray(da.choose(indices, datasets), dims=dims, attrs=attrs) + return weighted_array + + +def set_weights_to_zero_where_invalid(datasets, weights): + """Go through the weights and set to zero where corresponding datasets have a value equals _FillValue or nan.""" for i, dataset in enumerate(datasets): try: weights[i] = xr.where(dataset == dataset.attrs["_FillValue"], 0, weights[i]) except KeyError: weights[i] = xr.where(dataset.isnull(), 0, weights[i]) - indices = da.argmax(da.dstack(weights), axis=-1) - dims = datasets[0].dims - attrs = datasets[0].attrs - weighted_array = xr.DataArray(da.choose(indices, datasets), - dims=dims, attrs=attrs) - return weighted_array + return weights + + +def get_combined_start_end_times(*metadata_objects): + """Get the start and end times attributes valid for the entire dataset series.""" + start_time = datetime.now() + for md_obj in metadata_objects: + if md_obj['start_time'] < start_time: + start_time = md_obj['start_time'] + + end_time = datetime.fromtimestamp(0) + for md_obj in metadata_objects: + if md_obj['end_time'] > end_time: + end_time = md_obj['end_time'] + + return start_time, end_time def timeseries(datasets): diff --git a/satpy/tests/multiscene_tests/test_blend.py b/satpy/tests/multiscene_tests/test_blend.py index 8153067bc0..1abf6dd693 100644 --- a/satpy/tests/multiscene_tests/test_blend.py +++ b/satpy/tests/multiscene_tests/test_blend.py @@ -27,6 +27,7 @@ from pyresample.geometry import AreaDefinition from satpy import DataQuery +from satpy.multiscene import stack from satpy.tests.multiscene_tests.test_utils import _create_test_area, _create_test_dataset, _create_test_int8_dataset from satpy.tests.utils import make_dataid @@ -52,6 +53,16 @@ def scene1_with_weights(self): modifiers=() ) scene[dsid1] = _create_test_int8_dataset(name='geo-ct', area=area, values=1) + scene[dsid1].attrs['platform_name'] = 'Meteosat-11' + scene[dsid1].attrs['sensor'] = set({'seviri'}) + scene[dsid1].attrs['units'] = '1' + scene[dsid1].attrs['long_name'] = 'NWC GEO CT Cloud Type' + scene[dsid1].attrs['orbital_parameters'] = {'satellite_nominal_altitude': 35785863.0, + 'satellite_nominal_longitude': 0.0, + 'satellite_nominal_latitude': 0} + scene[dsid1].attrs['start_time'] = datetime(2023, 1, 16, 11, 9, 17) + scene[dsid1].attrs['end_time'] = datetime(2023, 1, 16, 11, 12, 22) + wgt1 = _create_test_dataset(name='geo-ct-wgt', area=area, values=0) wgt1[self._line, :] = 2 @@ -63,6 +74,9 @@ def scene1_with_weights(self): modifiers=() ) scene[dsid2] = _create_test_int8_dataset(name='geo-cma', area=area, values=2) + scene[dsid2].attrs['start_time'] = datetime(2023, 1, 16, 11, 9, 17) + scene[dsid2].attrs['end_time'] = datetime(2023, 1, 16, 11, 12, 22) + wgt2 = _create_test_dataset(name='geo-cma-wgt', area=area, values=0) return scene, [wgt1, wgt2] @@ -76,17 +90,29 @@ def scene2_with_weights(self): scene = Scene() dsid1 = make_dataid( name="polar-ct", + resolution=1000, modifiers=() ) scene[dsid1] = _create_test_int8_dataset(name='polar-ct', area=area, values=3) + scene[dsid1].attrs['platform_name'] = 'NOAA-18' + scene[dsid1].attrs['sensor'] = set({'avhrr-3'}) + scene[dsid1].attrs['units'] = '1' + scene[dsid1].attrs['long_name'] = 'SAFNWC PPS CT Cloud Type' scene[dsid1][-1, :] = scene[dsid1].attrs['_FillValue'] + scene[dsid1].attrs['start_time'] = datetime(2023, 1, 16, 11, 12, 57, 500000) + scene[dsid1].attrs['end_time'] = datetime(2023, 1, 16, 11, 28, 1, 900000) + wgt1 = _create_test_dataset(name='polar-ct-wgt', area=area, values=1) dsid2 = make_dataid( name="polar-cma", + resolution=1000, modifiers=() ) scene[dsid2] = _create_test_int8_dataset(name='polar-cma', area=area, values=4) + scene[dsid2].attrs['start_time'] = datetime(2023, 1, 16, 11, 12, 57, 500000) + scene[dsid2].attrs['end_time'] = datetime(2023, 1, 16, 11, 28, 1, 900000) + wgt2 = _create_test_dataset(name='polar-cma-wgt', area=area, values=1) return scene, [wgt1, wgt2] @@ -110,8 +136,6 @@ def groups(self): def test_blend_two_scenes_using_stack(self, multi_scene_and_weights, groups, scene1_with_weights, scene2_with_weights): """Test blending two scenes by stacking them on top of each other using function 'stack'.""" - from satpy.multiscene import stack - multi_scene, weights = multi_scene_and_weights scene1, weights1 = scene1_with_weights scene2, weights2 = scene2_with_weights @@ -120,18 +144,32 @@ def test_blend_two_scenes_using_stack(self, multi_scene_and_weights, groups, resampled = multi_scene stacked = resampled.blend(blend_function=stack) + result = stacked['CloudType'].compute() expected = scene2['polar-ct'].copy() expected[-1, :] = scene1['geo-ct'][-1, :] - xr.testing.assert_equal(stacked['CloudType'].compute(), expected.compute()) + xr.testing.assert_equal(result, expected.compute()) + assert result.attrs['platform_name'] == 'Meteosat-11' + assert result.attrs['sensor'] == set({'seviri'}) + assert result.attrs['long_name'] == 'NWC GEO CT Cloud Type' + assert result.attrs['units'] == '1' + assert result.attrs['name'] == 'CloudType' + assert result.attrs['_FillValue'] == 255 + assert result.attrs['valid_range'] == [1, 15] + + assert result.attrs['start_time'] == datetime(2023, 1, 16, 11, 9, 17) + assert result.attrs['end_time'] == datetime(2023, 1, 16, 11, 12, 22) def test_blend_two_scenes_using_stack_weighted(self, multi_scene_and_weights, groups, scene1_with_weights, scene2_with_weights): - """Test stacking two scenes using weights.""" - from functools import partial + """Test stacking two scenes using weights - testing that metadata are combined correctly. - from satpy.multiscene import stack + Here we test that the start and end times can be combined so that they + describe the start and times of the entire data series. + + """ + from functools import partial multi_scene, weights = multi_scene_and_weights scene1, weights1 = scene1_with_weights @@ -152,6 +190,50 @@ def test_blend_two_scenes_using_stack_weighted(self, multi_scene_and_weights, gr result = weighted_blend['CloudType'].compute() xr.testing.assert_equal(result, expected.compute()) + expected_area = _create_test_area() + assert result.attrs['area'] == expected_area + assert 'sensor' not in result.attrs + assert 'platform_name' not in result.attrs + assert 'long_name' not in result.attrs + assert result.attrs['units'] == '1' + assert result.attrs['name'] == 'CloudType' + assert result.attrs['_FillValue'] == 255 + assert result.attrs['valid_range'] == [1, 15] + + assert result.attrs['start_time'] == datetime(2023, 1, 16, 11, 9, 17) + assert result.attrs['end_time'] == datetime(2023, 1, 16, 11, 28, 1, 900000) + + def test_blend_two_scenes_using_stack_weighted_no_time_combination(self, multi_scene_and_weights, groups, + scene1_with_weights, scene2_with_weights): + """Test stacking two scenes using weights - test that the start and end times are averaged and not combined.""" + from functools import partial + + multi_scene, weights = multi_scene_and_weights + scene1, weights1 = scene1_with_weights + scene2, weights2 = scene2_with_weights + + simple_groups = {DataQuery(name='CloudType'): groups[DataQuery(name='CloudType')]} + multi_scene.group(simple_groups) + + weights = [weights[0][0], weights[1][0]] + stack_with_weights = partial(stack, weights=weights, combine_times=False) + weighted_blend = multi_scene.blend(blend_function=stack_with_weights) + + result = weighted_blend['CloudType'].compute() + + expected_area = _create_test_area() + assert result.attrs['area'] == expected_area + assert 'sensor' not in result.attrs + assert 'platform_name' not in result.attrs + assert 'long_name' not in result.attrs + assert result.attrs['units'] == '1' + assert result.attrs['name'] == 'CloudType' + assert result.attrs['_FillValue'] == 255 + assert result.attrs['valid_range'] == [1, 15] + + assert result.attrs['start_time'] == datetime(2023, 1, 16, 11, 11, 7, 250000) + assert result.attrs['end_time'] == datetime(2023, 1, 16, 11, 20, 11, 950000) + @pytest.fixture def datasets_and_weights(self): """X-Array datasets with area definition plus weights for input to tests.""" @@ -192,6 +274,7 @@ def test_blend_function_stack_weighted(self, datasets_and_weights, line, column) """Test the 'stack_weighted' function.""" from functools import partial + from satpy.dataset import combine_metadata from satpy.multiscene import stack input_data = datasets_and_weights @@ -199,7 +282,7 @@ def test_blend_function_stack_weighted(self, datasets_and_weights, line, column) input_data['weights'][1][line, :] = 2 input_data['weights'][2][:, column] = 2 - stack_with_weights = partial(stack, weights=input_data['weights']) + stack_with_weights = partial(stack, weights=input_data['weights'], combine_times=False) blend_result = stack_with_weights(input_data['datasets'][0:3]) ds1 = input_data['datasets'][0] @@ -208,8 +291,10 @@ def test_blend_function_stack_weighted(self, datasets_and_weights, line, column) expected = ds1.copy() expected[:, column] = ds3[:, column] expected[line, :] = ds2[line, :] + expected.attrs = combine_metadata(*[x.attrs for x in input_data['datasets'][0:3]]) xr.testing.assert_equal(blend_result.compute(), expected.compute()) + assert expected.attrs == blend_result.attrs def test_blend_function_stack(self, datasets_and_weights): @@ -225,10 +310,6 @@ def test_blend_function_stack(self, datasets_and_weights): expected = ds2.copy() xr.testing.assert_equal(res.compute(), expected.compute()) - # assert expected.attrs == res.attrs - # FIXME! Looks like the attributes are taken from the first dataset. Should - # be like that? So in this case the datetime is different from "expected" - # (= in this case the last dataset in the stack, the one on top) def test_timeseries(self, datasets_and_weights): """Test the 'timeseries' function.""" From 5e913837a196a5ce71462bc86bd713925ed101d5 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Thu, 26 Jan 2023 16:08:19 +0100 Subject: [PATCH 22/30] Add a bit of documenation on using weights when blending with the stack function Signed-off-by: Adam.Dybbroe --- doc/source/multiscene.rst | 43 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/doc/source/multiscene.rst b/doc/source/multiscene.rst index 8a7be6b8aa..163142af5c 100644 --- a/doc/source/multiscene.rst +++ b/doc/source/multiscene.rst @@ -85,6 +85,49 @@ iteratively overlays the remaining datasets on top. >>> blended_scene = new_mscn.blend() >>> blended_scene.save_datasets() + +Stacking scenes using weights +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +It is also possible to blend scenes together in a bit more sophisticated manner +using pixel based weighting instead of just stacking the scenes on top of each +other as described above. This can for instance be useful to make a cloud +parameter (cover, height, etc) composite combining cloud parameters derived +from both geostationary and polar orbiting satellite data over aa given area +(for instance at high latitudes where geostatioonary data degrade quickly with +latitude and polar data are more frequent) scenes valid close in time to each +other. + +This weighted blending can be accomplished via the use of the builtin +:func:`~functools.partial` function (see `Partial +`_) and the +default :func:`~satpy.multiscene.stack` function. The +:func:`~satpy.multiscene.stack` function can take the optional argument +`weights` (`None` on default) which should be a sequence (of length equal to +the number of scenes being blended) of arrays with pixel weights. + +The code below gives an example of how two clouud scenes can be blended using +the satellite zenith angles to weight which pixels to take from each of the two +scenes. The idea being that the reliability of the cloud parameter is higher +when the satellite zenith angle is small. + + >>> from satpy import Scene, MultiScene, DataQuery + >>> from functools import partial + >>> geo_scene = Scene(filenames=glob('/data/to/nwcsaf/geo/files/*nc'), reader='nwcsaf-geo') + >>> geo_scene.load(['ct']) + >>> polar_scene = Scene(filenames=glob('/data/to/nwcsaf/pps/noaa18/files/*nc'), reader='nwcsaf-pps_nc') + >>> polar_scene.load(['cma', 'ct']) + >>> mscn = MultiScene([geo_scene, polar_scene]) + >>> groups = {DataQuery(name='CTY_group'): ['ct']} + >>> mscn.group(groups) + >>> resampled = mscn.resample(areaid, reduce_data=False) + >>> weights = [1./geo_satz, 1./n18_satz] + >>> stack_with_weights = partial(stack, weights=weights) + >>> blended = resampled.blend(blend_function=stack_with_weights) + >>> blended_scene.save_datasets() + + + Grouping Similar Datasets ^^^^^^^^^^^^^^^^^^^^^^^^^ From 897df3d334c3986115a16f754d6b4e57451ea2e9 Mon Sep 17 00:00:00 2001 From: Adam Dybbroe Date: Fri, 27 Jan 2023 10:24:23 +0100 Subject: [PATCH 23/30] Improve formulation on the motive of blending geo and polar scenes Co-authored-by: David Hoese --- doc/source/multiscene.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/source/multiscene.rst b/doc/source/multiscene.rst index 163142af5c..9127a8d5fc 100644 --- a/doc/source/multiscene.rst +++ b/doc/source/multiscene.rst @@ -93,10 +93,10 @@ It is also possible to blend scenes together in a bit more sophisticated manner using pixel based weighting instead of just stacking the scenes on top of each other as described above. This can for instance be useful to make a cloud parameter (cover, height, etc) composite combining cloud parameters derived -from both geostationary and polar orbiting satellite data over aa given area -(for instance at high latitudes where geostatioonary data degrade quickly with -latitude and polar data are more frequent) scenes valid close in time to each -other. +from both geostationary and polar orbiting satellite data close in time and +over a given area. This is useful for instance at high latitudes where +geostationary data degrade quickly with latitude and polar data are more +frequent. This weighted blending can be accomplished via the use of the builtin :func:`~functools.partial` function (see `Partial From 0b187ddfc927c1b52815a2b35b4d0ea66a45be2f Mon Sep 17 00:00:00 2001 From: Adam Dybbroe Date: Fri, 27 Jan 2023 10:24:43 +0100 Subject: [PATCH 24/30] Fix typo Co-authored-by: David Hoese --- doc/source/multiscene.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/multiscene.rst b/doc/source/multiscene.rst index 9127a8d5fc..3484f898d9 100644 --- a/doc/source/multiscene.rst +++ b/doc/source/multiscene.rst @@ -106,7 +106,7 @@ default :func:`~satpy.multiscene.stack` function. The `weights` (`None` on default) which should be a sequence (of length equal to the number of scenes being blended) of arrays with pixel weights. -The code below gives an example of how two clouud scenes can be blended using +The code below gives an example of how two cloud scenes can be blended using the satellite zenith angles to weight which pixels to take from each of the two scenes. The idea being that the reliability of the cloud parameter is higher when the satellite zenith angle is small. From c987b5dfcd6be56bff1280c78d40919ec09e3987 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 27 Jan 2023 10:34:35 +0100 Subject: [PATCH 25/30] Refactor function using one for loop for efficiency and make private Signed-off-by: Adam.Dybbroe --- satpy/multiscene.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/satpy/multiscene.py b/satpy/multiscene.py index 22d566d1ff..dcd63d5fbe 100644 --- a/satpy/multiscene.py +++ b/satpy/multiscene.py @@ -77,7 +77,7 @@ def _stack_weighted(datasets, weights, combine_times): attrs = combine_metadata(*[x.attrs for x in datasets]) if combine_times: - attrs['start_time'], attrs['end_time'] = get_combined_start_end_times(*[x.attrs for x in datasets]) + attrs['start_time'], attrs['end_time'] = _get_combined_start_end_times(*[x.attrs for x in datasets]) dims = datasets[0].dims weighted_array = xr.DataArray(da.choose(indices, datasets), dims=dims, attrs=attrs) @@ -95,15 +95,13 @@ def set_weights_to_zero_where_invalid(datasets, weights): return weights -def get_combined_start_end_times(*metadata_objects): +def _get_combined_start_end_times(*metadata_objects): """Get the start and end times attributes valid for the entire dataset series.""" start_time = datetime.now() + end_time = datetime.fromtimestamp(0) for md_obj in metadata_objects: if md_obj['start_time'] < start_time: start_time = md_obj['start_time'] - - end_time = datetime.fromtimestamp(0) - for md_obj in metadata_objects: if md_obj['end_time'] > end_time: end_time = md_obj['end_time'] From 933de7bd9a03ab2efd510ddc82e7e8080b8cb632 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 27 Jan 2023 11:03:14 +0100 Subject: [PATCH 26/30] Fix code example for blending with weihts - getting the area-id correctly Signed-off-by: Adam.Dybbroe --- doc/source/multiscene.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/source/multiscene.rst b/doc/source/multiscene.rst index 3484f898d9..5a6b15397e 100644 --- a/doc/source/multiscene.rst +++ b/doc/source/multiscene.rst @@ -100,7 +100,7 @@ frequent. This weighted blending can be accomplished via the use of the builtin :func:`~functools.partial` function (see `Partial -`_) and the + `_) and the default :func:`~satpy.multiscene.stack` function. The :func:`~satpy.multiscene.stack` function can take the optional argument `weights` (`None` on default) which should be a sequence (of length equal to @@ -113,6 +113,8 @@ when the satellite zenith angle is small. >>> from satpy import Scene, MultiScene, DataQuery >>> from functools import partial + >>> from satpy.resample import get_area_def + >>> areaid = get_area_def("myarea") >>> geo_scene = Scene(filenames=glob('/data/to/nwcsaf/geo/files/*nc'), reader='nwcsaf-geo') >>> geo_scene.load(['ct']) >>> polar_scene = Scene(filenames=glob('/data/to/nwcsaf/pps/noaa18/files/*nc'), reader='nwcsaf-pps_nc') From 3c125dc07a9f44eee372fcb1706a44d631a3acf2 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 27 Jan 2023 11:36:11 +0100 Subject: [PATCH 27/30] Only try combine start and end times if they are already there Signed-off-by: Adam.Dybbroe --- satpy/multiscene.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/satpy/multiscene.py b/satpy/multiscene.py index dcd63d5fbe..72b7f95e7f 100644 --- a/satpy/multiscene.py +++ b/satpy/multiscene.py @@ -77,7 +77,8 @@ def _stack_weighted(datasets, weights, combine_times): attrs = combine_metadata(*[x.attrs for x in datasets]) if combine_times: - attrs['start_time'], attrs['end_time'] = _get_combined_start_end_times(*[x.attrs for x in datasets]) + if 'start_time' in attrs and 'end_time' in attrs: + attrs['start_time'], attrs['end_time'] = _get_combined_start_end_times(*[x.attrs for x in datasets]) dims = datasets[0].dims weighted_array = xr.DataArray(da.choose(indices, datasets), dims=dims, attrs=attrs) @@ -85,7 +86,7 @@ def _stack_weighted(datasets, weights, combine_times): def set_weights_to_zero_where_invalid(datasets, weights): - """Go through the weights and set to zero where corresponding datasets have a value equals _FillValue or nan.""" + """Go through the weights and set to pixel values to zero where corresponding datasets are invalid.""" for i, dataset in enumerate(datasets): try: weights[i] = xr.where(dataset == dataset.attrs["_FillValue"], 0, weights[i]) From 238c71cf6f2d7d3dc22d8ac470c3e6286ce2c277 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 27 Jan 2023 11:36:54 +0100 Subject: [PATCH 28/30] Revert back on standard_name for cloudtype dataset Signed-off-by: Adam.Dybbroe --- satpy/etc/readers/nwcsaf-pps_nc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/etc/readers/nwcsaf-pps_nc.yaml b/satpy/etc/readers/nwcsaf-pps_nc.yaml index 1b673125d6..8ae3b4ae7a 100644 --- a/satpy/etc/readers/nwcsaf-pps_nc.yaml +++ b/satpy/etc/readers/nwcsaf-pps_nc.yaml @@ -120,7 +120,7 @@ datasets: name: ct file_type: nc_nwcsaf_ct coordinates: [lon, lat] - standard_name: ct + standard_name: cloudtype ct_conditions: name: ct_conditions From c76e790b037d57bb73c202eb8f583960f55eef64 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 27 Jan 2023 11:37:51 +0100 Subject: [PATCH 29/30] Be more explicit in the doc pages when storing the blended cloudtype scene Signed-off-by: Adam.Dybbroe --- doc/source/multiscene.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/multiscene.rst b/doc/source/multiscene.rst index 5a6b15397e..95c472123d 100644 --- a/doc/source/multiscene.rst +++ b/doc/source/multiscene.rst @@ -126,7 +126,7 @@ when the satellite zenith angle is small. >>> weights = [1./geo_satz, 1./n18_satz] >>> stack_with_weights = partial(stack, weights=weights) >>> blended = resampled.blend(blend_function=stack_with_weights) - >>> blended_scene.save_datasets() + >>> blended_scene.save_dataset('CTY_group', filename='./blended_stack_weighted_geo_polar.nc') From 8baca7910e09771933a04f5ffc38d2fdabc714cb Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Fri, 27 Jan 2023 16:03:48 +0100 Subject: [PATCH 30/30] Fix minor editorial in RTDs Signed-off-by: Adam.Dybbroe --- doc/source/multiscene.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/multiscene.rst b/doc/source/multiscene.rst index 95c472123d..35e48cbde1 100644 --- a/doc/source/multiscene.rst +++ b/doc/source/multiscene.rst @@ -100,7 +100,7 @@ frequent. This weighted blending can be accomplished via the use of the builtin :func:`~functools.partial` function (see `Partial - `_) and the +`_) and the default :func:`~satpy.multiscene.stack` function. The :func:`~satpy.multiscene.stack` function can take the optional argument `weights` (`None` on default) which should be a sequence (of length equal to