From 254c8f2dd542dbee2e3b1699fe1c74644cedbe6a Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 2 Mar 2022 14:12:31 -0600 Subject: [PATCH 1/5] Refactor VIIRS composite tests to use pytest for ERF DNB composite tests --- satpy/tests/compositor_tests/test_viirs.py | 61 ++++++++++++---------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/satpy/tests/compositor_tests/test_viirs.py b/satpy/tests/compositor_tests/test_viirs.py index 885279918c..492c565000 100644 --- a/satpy/tests/compositor_tests/test_viirs.py +++ b/satpy/tests/compositor_tests/test_viirs.py @@ -20,7 +20,7 @@ import unittest -class TestVIIRSComposites(unittest.TestCase): +class TestVIIRSCompositesUnitTest(unittest.TestCase): """Test VIIRS-specific composites.""" def test_load_composite_yaml(self): @@ -116,14 +116,14 @@ def test_adaptive_dnb(self): data = res.compute() np.testing.assert_allclose(data.data, 0.999, rtol=1e-4) - def test_erf_dnb(self): - """Test the 'dynamic_dnb' or ERF DNB compositor.""" + def test_hncc_dnb(self): + """Test the 'hncc_dnb' compositor.""" import dask.array as da import numpy as np import xarray as xr from pyresample.geometry import AreaDefinition - from satpy.composites.viirs import ERFDNB + from satpy.composites.viirs import NCCZinke rows = 5 cols = 10 area = AreaDefinition( @@ -133,11 +133,10 @@ def test_erf_dnb(self): cols, rows, (-20037508.34, -10018754.17, 20037508.34, 10018754.17)) - comp = ERFDNB('dynamic_dnb', prerequisites=('dnb',), - standard_name='toa_outgoing_radiance_per_' - 'unit_wavelength') + comp = NCCZinke('hncc_dnb', prerequisites=('dnb',), + standard_name='toa_outgoing_radiance_per_' + 'unit_wavelength') dnb = np.zeros((rows, cols)) + 0.25 - dnb[2, :cols // 2] = np.nan dnb[3, :] += 0.25 dnb[4:, :] += 0.5 dnb = da.from_array(dnb, chunks=25) @@ -164,26 +163,28 @@ def test_erf_dnb(self): res = comp((c01, c02, c03, mif)) self.assertIsInstance(res, xr.DataArray) self.assertIsInstance(res.data, da.Array) - self.assertEqual(res.attrs['name'], 'dynamic_dnb') + self.assertEqual(res.attrs['name'], 'hncc_dnb') self.assertEqual(res.attrs['standard_name'], - 'equalized_radiance') + 'ncc_radiance') data = res.compute() unique = np.unique(data) - assert np.isnan(unique).any() - nonnan_unique = unique[~np.isnan(unique)] np.testing.assert_allclose( - nonnan_unique, - [0.00000000e+00, 1.00446703e-01, 1.64116082e-01, 2.09233451e-01, - 1.43916324e+02, 2.03528498e+02, 2.49270516e+02]) + unique, [3.48479712e-04, 6.96955799e-04, 1.04543189e-03, 4.75394738e-03, + 9.50784532e-03, 1.42617433e-02, 1.50001560e+03, 3.00001560e+03, + 4.50001560e+03]) - def test_hncc_dnb(self): - """Test the 'hncc_dnb' compositor.""" + +class TestVIIRSComposites: + """Test various VIIRS-specific composites.""" + + def test_erf_dnb(self): + """Test the 'dynamic_dnb' or ERF DNB compositor.""" import dask.array as da import numpy as np import xarray as xr from pyresample.geometry import AreaDefinition - from satpy.composites.viirs import NCCZinke + from satpy.composites.viirs import ERFDNB rows = 5 cols = 10 area = AreaDefinition( @@ -193,10 +194,11 @@ def test_hncc_dnb(self): cols, rows, (-20037508.34, -10018754.17, 20037508.34, 10018754.17)) - comp = NCCZinke('hncc_dnb', prerequisites=('dnb',), - standard_name='toa_outgoing_radiance_per_' - 'unit_wavelength') + comp = ERFDNB('dynamic_dnb', prerequisites=('dnb',), + standard_name='toa_outgoing_radiance_per_' + 'unit_wavelength') dnb = np.zeros((rows, cols)) + 0.25 + dnb[2, :cols // 2] = np.nan dnb[3, :] += 0.25 dnb[4:, :] += 0.5 dnb = da.from_array(dnb, chunks=25) @@ -221,14 +223,15 @@ def test_hncc_dnb(self): dims=('y',), attrs={'name': 'moon_illumination_fraction', 'area': area}) res = comp((c01, c02, c03, mif)) - self.assertIsInstance(res, xr.DataArray) - self.assertIsInstance(res.data, da.Array) - self.assertEqual(res.attrs['name'], 'hncc_dnb') - self.assertEqual(res.attrs['standard_name'], - 'ncc_radiance') + assert isinstance(res, xr.DataArray) + assert isinstance(res.data, da.Array) + assert res.attrs['name'] == 'dynamic_dnb' + assert res.attrs['standard_name'] == 'equalized_radiance' data = res.compute() unique = np.unique(data) + assert np.isnan(unique).any() + nonnan_unique = unique[~np.isnan(unique)] np.testing.assert_allclose( - unique, [3.48479712e-04, 6.96955799e-04, 1.04543189e-03, 4.75394738e-03, - 9.50784532e-03, 1.42617433e-02, 1.50001560e+03, 3.00001560e+03, - 4.50001560e+03]) + nonnan_unique, + [0.00000000e+00, 1.00446703e-01, 1.64116082e-01, 2.09233451e-01, + 1.43916324e+02, 2.03528498e+02, 2.49270516e+02]) From 2fe1a7bc99b2546b8692ad7e08ca33891de45771 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 3 Mar 2022 08:33:14 -0600 Subject: [PATCH 2/5] Add units to ERF DNB tests --- satpy/tests/compositor_tests/test_viirs.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/satpy/tests/compositor_tests/test_viirs.py b/satpy/tests/compositor_tests/test_viirs.py index 492c565000..0028fc5927 100644 --- a/satpy/tests/compositor_tests/test_viirs.py +++ b/satpy/tests/compositor_tests/test_viirs.py @@ -19,6 +19,8 @@ import unittest +import pytest + class TestVIIRSCompositesUnitTest(unittest.TestCase): """Test VIIRS-specific composites.""" @@ -177,7 +179,8 @@ def test_hncc_dnb(self): class TestVIIRSComposites: """Test various VIIRS-specific composites.""" - def test_erf_dnb(self): + @pytest.mark.parametrize("dnb_units", ["W m-2 sr-1", "W cm-2 sr-1"]) + def test_erf_dnb(self, dnb_units): """Test the 'dynamic_dnb' or ERF DNB compositor.""" import dask.array as da import numpy as np @@ -201,10 +204,12 @@ def test_erf_dnb(self): dnb[2, :cols // 2] = np.nan dnb[3, :] += 0.25 dnb[4:, :] += 0.5 + if dnb_units == "W cm-2 sr-1": + dnb /= 10000.0 dnb = da.from_array(dnb, chunks=25) c01 = xr.DataArray(dnb, dims=('y', 'x'), - attrs={'name': 'DNB', 'area': area}) + attrs={'name': 'DNB', 'area': area, 'units': dnb_units}) sza = np.zeros((rows, cols)) + 70.0 sza[:, 3] += 20.0 sza[:, 4:] += 45.0 From 8e85941371aeef220111de9a252fef4949558e7d Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 3 Mar 2022 08:40:30 -0600 Subject: [PATCH 3/5] Fix ERF DNB normalization's saturation correction units handling --- satpy/composites/viirs.py | 2 +- satpy/tests/compositor_tests/test_viirs.py | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/satpy/composites/viirs.py b/satpy/composites/viirs.py index 3409ff1154..249a9a0cd5 100644 --- a/satpy/composites/viirs.py +++ b/satpy/composites/viirs.py @@ -268,7 +268,7 @@ def _saturation_correction(self, dnb_data, unit_factor, min_val, max_val)) / dnb_data.size LOG.debug("Dynamic DNB saturation percentage: %f", saturation_pct) while saturation_pct > 0.005: - max_val *= 1.1 * unit_factor + max_val *= 1.1 saturation_pct = float(np.count_nonzero( dnb_data > max_val)) / dnb_data.size LOG.debug("Dynamic DNB saturation percentage: %f", diff --git a/satpy/tests/compositor_tests/test_viirs.py b/satpy/tests/compositor_tests/test_viirs.py index 0028fc5927..ca276926c0 100644 --- a/satpy/tests/compositor_tests/test_viirs.py +++ b/satpy/tests/compositor_tests/test_viirs.py @@ -180,7 +180,8 @@ class TestVIIRSComposites: """Test various VIIRS-specific composites.""" @pytest.mark.parametrize("dnb_units", ["W m-2 sr-1", "W cm-2 sr-1"]) - def test_erf_dnb(self, dnb_units): + @pytest.mark.parametrize("saturation_correction", [False, True]) + def test_erf_dnb(self, dnb_units, saturation_correction): """Test the 'dynamic_dnb' or ERF DNB compositor.""" import dask.array as da import numpy as np @@ -198,6 +199,7 @@ def test_erf_dnb(self, dnb_units): (-20037508.34, -10018754.17, 20037508.34, 10018754.17)) comp = ERFDNB('dynamic_dnb', prerequisites=('dnb',), + saturation_correction=saturation_correction, standard_name='toa_outgoing_radiance_per_' 'unit_wavelength') dnb = np.zeros((rows, cols)) + 0.25 @@ -236,7 +238,12 @@ def test_erf_dnb(self, dnb_units): unique = np.unique(data) assert np.isnan(unique).any() nonnan_unique = unique[~np.isnan(unique)] - np.testing.assert_allclose( - nonnan_unique, - [0.00000000e+00, 1.00446703e-01, 1.64116082e-01, 2.09233451e-01, - 1.43916324e+02, 2.03528498e+02, 2.49270516e+02]) + if saturation_correction: + exp_unique = [0.000000e+00, 3.978305e-04, 6.500003e-04, + 8.286927e-04, 5.628335e-01, 7.959671e-01, + 9.748567e-01] + else: + exp_unique = [0.00000000e+00, 1.00446703e-01, 1.64116082e-01, + 2.09233451e-01, 1.43916324e+02, 2.03528498e+02, + 2.49270516e+02] + np.testing.assert_allclose(nonnan_unique, exp_unique) From ac14571e92a7ba689d77f92bd39ac88a8700a15f Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 3 Mar 2022 09:07:45 -0600 Subject: [PATCH 4/5] Convert all VIIRS composite tests to pytest --- satpy/tests/compositor_tests/test_viirs.py | 37 ++++++++-------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/satpy/tests/compositor_tests/test_viirs.py b/satpy/tests/compositor_tests/test_viirs.py index ca276926c0..3fa6e03dd3 100644 --- a/satpy/tests/compositor_tests/test_viirs.py +++ b/satpy/tests/compositor_tests/test_viirs.py @@ -17,13 +17,11 @@ # satpy. If not, see . """Tests for VIIRS compositors.""" -import unittest - import pytest -class TestVIIRSCompositesUnitTest(unittest.TestCase): - """Test VIIRS-specific composites.""" +class TestVIIRSComposites: + """Test various VIIRS-specific composites.""" def test_load_composite_yaml(self): """Test loading the yaml for this sensor.""" @@ -66,11 +64,10 @@ def test_histogram_dnb(self): dims=('y', 'x'), attrs={'name': 'solar_zenith_angle', 'area': area}) res = comp((c01, c02)) - self.assertIsInstance(res, xr.DataArray) - self.assertIsInstance(res.data, da.Array) - self.assertEqual(res.attrs['name'], 'histogram_dnb') - self.assertEqual(res.attrs['standard_name'], - 'equalized_radiance') + assert isinstance(res, xr.DataArray) + assert isinstance(res.data, da.Array) + assert res.attrs['name'] == 'histogram_dnb' + assert res.attrs['standard_name'] == 'equalized_radiance' data = res.compute() unique_values = np.unique(data) np.testing.assert_allclose(unique_values, [0.5994, 0.7992, 0.999], rtol=1e-3) @@ -110,11 +107,10 @@ def test_adaptive_dnb(self): dims=('y', 'x'), attrs={'name': 'solar_zenith_angle', 'area': area}) res = comp((c01, c02)) - self.assertIsInstance(res, xr.DataArray) - self.assertIsInstance(res.data, da.Array) - self.assertEqual(res.attrs['name'], 'adaptive_dnb') - self.assertEqual(res.attrs['standard_name'], - 'equalized_radiance') + assert isinstance(res, xr.DataArray) + assert isinstance(res.data, da.Array) + assert res.attrs['name'] == 'adaptive_dnb' + assert res.attrs['standard_name'] == 'equalized_radiance' data = res.compute() np.testing.assert_allclose(data.data, 0.999, rtol=1e-4) @@ -163,11 +159,10 @@ def test_hncc_dnb(self): dims=('y',), attrs={'name': 'moon_illumination_fraction', 'area': area}) res = comp((c01, c02, c03, mif)) - self.assertIsInstance(res, xr.DataArray) - self.assertIsInstance(res.data, da.Array) - self.assertEqual(res.attrs['name'], 'hncc_dnb') - self.assertEqual(res.attrs['standard_name'], - 'ncc_radiance') + assert isinstance(res, xr.DataArray) + assert isinstance(res.data, da.Array) + assert res.attrs['name'] == 'hncc_dnb' + assert res.attrs['standard_name'] == 'ncc_radiance' data = res.compute() unique = np.unique(data) np.testing.assert_allclose( @@ -175,10 +170,6 @@ def test_hncc_dnb(self): 9.50784532e-03, 1.42617433e-02, 1.50001560e+03, 3.00001560e+03, 4.50001560e+03]) - -class TestVIIRSComposites: - """Test various VIIRS-specific composites.""" - @pytest.mark.parametrize("dnb_units", ["W m-2 sr-1", "W cm-2 sr-1"]) @pytest.mark.parametrize("saturation_correction", [False, True]) def test_erf_dnb(self, dnb_units, saturation_correction): From 1e1d54903cbc2bda7c676b7a80a2ff59bcd12f2d Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 3 Mar 2022 09:09:48 -0600 Subject: [PATCH 5/5] Refactor viirs composites test to move shared imports to module level --- satpy/tests/compositor_tests/test_viirs.py | 24 ++++------------------ 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/satpy/tests/compositor_tests/test_viirs.py b/satpy/tests/compositor_tests/test_viirs.py index 3fa6e03dd3..820bd40d38 100644 --- a/satpy/tests/compositor_tests/test_viirs.py +++ b/satpy/tests/compositor_tests/test_viirs.py @@ -17,7 +17,11 @@ # satpy. If not, see . """Tests for VIIRS compositors.""" +import dask.array as da +import numpy as np import pytest +import xarray as xr +from pyresample.geometry import AreaDefinition class TestVIIRSComposites: @@ -30,11 +34,6 @@ def test_load_composite_yaml(self): def test_histogram_dnb(self): """Test the 'histogram_dnb' compositor.""" - import dask.array as da - import numpy as np - import xarray as xr - from pyresample.geometry import AreaDefinition - from satpy.composites.viirs import HistogramDNB rows = 5 cols = 10 @@ -74,11 +73,6 @@ def test_histogram_dnb(self): def test_adaptive_dnb(self): """Test the 'adaptive_dnb' compositor.""" - import dask.array as da - import numpy as np - import xarray as xr - from pyresample.geometry import AreaDefinition - from satpy.composites.viirs import AdaptiveDNB rows = 5 cols = 10 @@ -116,11 +110,6 @@ def test_adaptive_dnb(self): def test_hncc_dnb(self): """Test the 'hncc_dnb' compositor.""" - import dask.array as da - import numpy as np - import xarray as xr - from pyresample.geometry import AreaDefinition - from satpy.composites.viirs import NCCZinke rows = 5 cols = 10 @@ -174,11 +163,6 @@ def test_hncc_dnb(self): @pytest.mark.parametrize("saturation_correction", [False, True]) def test_erf_dnb(self, dnb_units, saturation_correction): """Test the 'dynamic_dnb' or ERF DNB compositor.""" - import dask.array as da - import numpy as np - import xarray as xr - from pyresample.geometry import AreaDefinition - from satpy.composites.viirs import ERFDNB rows = 5 cols = 10