From 826122a2dd67b4332b28bfcdf945c6dc7e9cc565 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Sun, 21 Mar 2021 09:47:15 -0500 Subject: [PATCH 01/29] Add reader_kwarg to omit limb correction on ATMS sensors. --- satpy/readers/mirs.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/satpy/readers/mirs.py b/satpy/readers/mirs.py index 10934e1281..d2d4a88531 100644 --- a/satpy/readers/mirs.py +++ b/satpy/readers/mirs.py @@ -190,13 +190,29 @@ def limb_correct_atms_bt(bt_data, surf_type_mask, coeff_fns, ds_info): class MiRSL2ncHandler(BaseFileHandler): - """MiRS handler for NetCDF4 files using xarray.""" + """MiRS handler for NetCDF4 files using xarray. - def __init__(self, filename, filename_info, filetype_info): + The MiRS retrieval algorithm runs on multiple + sensors. For the ATMS sensors, a limb correction + is applied by default. In order to change that + behavior, use the keyword argument limb_correction=False + + + from satpy import Scene, find_files_and_readers + + filenames = find_files_and_readers(base_dir, reader="mirs") + scene = satpy.Scene(filenames, + reader_kwargs={'limb_correction': False}) + + """ + + def __init__(self, filename, filename_info, filetype_info, + limb_correction=True): """Init method.""" super(MiRSL2ncHandler, self).__init__(filename, filename_info, - filetype_info) + filetype_info, + ) self.nc = xr.open_dataset(self.filename, decode_cf=True, @@ -212,6 +228,7 @@ def __init__(self, filename, filename_info, filetype_info): self.platform_name = self._get_platform_name self.sensor = self._get_sensor + self.limb_correction = limb_correction @property def platform_shortname(self): @@ -346,15 +363,15 @@ def get_dataset(self, ds_id, ds_info): data = self['BT'] data = data.rename(new_name_or_name_dict=ds_info["name"]) - if self.sensor.lower() != "atms": - LOG.info("Limb Correction will not be applied to non-ATMS BTs") - data = data[:, :, idx] - else: + if self.sensor.lower() == "atms" and self.limb_correction: sfc_type_mask = self['Sfc_type'] data = limb_correct_atms_bt(data, sfc_type_mask, self._get_coeff_filenames, ds_info) self.nc = self.nc.merge(data) + else: + LOG.info("No Limb Correction applied.") + data = data[:, :, idx] else: data = self[ds_id['name']] From 79218d767af1699ad02a28c18d367fc8a5e5492f Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Mon, 22 Mar 2021 05:31:29 -0500 Subject: [PATCH 02/29] Fix docstring in mirs reader class --- satpy/readers/mirs.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/satpy/readers/mirs.py b/satpy/readers/mirs.py index d2d4a88531..ea11c7c4a9 100644 --- a/satpy/readers/mirs.py +++ b/satpy/readers/mirs.py @@ -198,11 +198,10 @@ class MiRSL2ncHandler(BaseFileHandler): behavior, use the keyword argument limb_correction=False - from satpy import Scene, find_files_and_readers + from satpy import Scene, find_files_and_readers - filenames = find_files_and_readers(base_dir, reader="mirs") - scene = satpy.Scene(filenames, - reader_kwargs={'limb_correction': False}) + filenames = find_files_and_readers(base_dir, reader="mirs") + scene = Scene(filenames, reader_kwargs={'limb_correction': False}) """ From 1cdbc0c3d0b794708523d813a666987c8a3ffe73 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Mon, 22 Mar 2021 16:12:07 -0500 Subject: [PATCH 03/29] add check for calling limb correction only when sensor is atms --- satpy/tests/reader_tests/test_mirs.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/satpy/tests/reader_tests/test_mirs.py b/satpy/tests/reader_tests/test_mirs.py index bca09d3f90..1bc70e7918 100644 --- a/satpy/tests/reader_tests/test_mirs.py +++ b/satpy/tests/reader_tests/test_mirs.py @@ -82,7 +82,7 @@ def fake_coeff_from_fn(fn): return coeff_str -def _get_datasets_with_attributes(): +def _get_datasets_with_attributes(**kwargs): """Represent files with two resolution of variables in them (ex. OCEAN).""" bt = xr.DataArray(np.linspace(1830, 3930, N_SCANLINE * N_FOV * N_CHANNEL). reshape(N_SCANLINE, N_FOV, N_CHANNEL), @@ -91,7 +91,8 @@ def _get_datasets_with_attributes(): 'coordinates': "Longitude Latitude Freq", 'scale_factor': 0.01, '_FillValue': -999, - 'valid_range': [0, 50000]}) + 'valid_range': [0, 50000]}, + dims=('Scanline', 'Field_of_view', 'Channel')) rr = xr.DataArray(np.random.randint(100, 500, size=(N_SCANLINE, N_FOV)), attrs={'long_name': "Rain Rate (mm/hr)", 'units': "mm/hr", @@ -275,8 +276,14 @@ def test_basic_load(self, filenames, loadable_ids, platform_name): fd.side_effect = fake_coeff_from_fn loaded_data_arrs = r.load(loadable_ids) assert loaded_data_arrs + for data_id, data_arr in loaded_data_arrs.items(): if data_id['name'] not in ['latitude', 'longitude']: self._check_area(data_arr) self._check_fill(data_arr) self._check_attrs(data_arr, platform_name) + + if data_arr.attrs['sensor'] == 'atms': + fd.assert_called() + else: + fd.assert_not_called() From e783a8c8a7d554ce6a480a4490a84ad3d28657b9 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Sat, 27 Mar 2021 01:03:59 -0500 Subject: [PATCH 04/29] Fix docstring --- satpy/readers/mirs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/satpy/readers/mirs.py b/satpy/readers/mirs.py index ea11c7c4a9..a8dab41b26 100644 --- a/satpy/readers/mirs.py +++ b/satpy/readers/mirs.py @@ -195,13 +195,13 @@ class MiRSL2ncHandler(BaseFileHandler): The MiRS retrieval algorithm runs on multiple sensors. For the ATMS sensors, a limb correction is applied by default. In order to change that - behavior, use the keyword argument limb_correction=False + behavior, use the keyword argument ``limb_correction=False``:: - from satpy import Scene, find_files_and_readers + from satpy import Scene, find_files_and_readers - filenames = find_files_and_readers(base_dir, reader="mirs") - scene = Scene(filenames, reader_kwargs={'limb_correction': False}) + filenames = find_files_and_readers(base_dir, reader="mirs") + scene = Scene(filenames, reader_kwargs={'limb_correction': False}) """ From fc610c8442c0e23c30c34a97285694ca02b44337 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Sat, 27 Mar 2021 10:22:21 -0500 Subject: [PATCH 05/29] Add a kwarg test for limb_correction. --- satpy/tests/reader_tests/test_mirs.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/satpy/tests/reader_tests/test_mirs.py b/satpy/tests/reader_tests/test_mirs.py index 1bc70e7918..0ec614845b 100644 --- a/satpy/tests/reader_tests/test_mirs.py +++ b/satpy/tests/reader_tests/test_mirs.py @@ -287,3 +287,28 @@ def test_basic_load(self, filenames, loadable_ids, platform_name): fd.assert_called() else: fd.assert_not_called() + + @pytest.mark.parametrize( + ("filenames", "loadable_ids", "platform_name"), + [ + ([AWIPS_FILE], TEST_VARS, "metop-a"), + ([NPP_MIRS_L2_SWATH], TEST_VARS, "npp"), + ([OTHER_MIRS_L2_SWATH], TEST_VARS, "gpm"), + ] + ) + def test_kwarg_load(self, filenames, loadable_ids, platform_name): + """Test the limb_correction kwarg when filehandler is loaded.""" + from satpy.readers import load_reader + with mock.patch('satpy.readers.mirs.xr.open_dataset') as od: + od.side_effect = fake_open_dataset + r = load_reader(self.reader_configs) + loadables = r.select_files_from_pathnames(filenames) + r.create_filehandlers(loadables, fh_kwargs={"limb_correction": False}) + + with mock.patch('satpy.readers.mirs.read_atms_coeff_to_string') as \ + fd, mock.patch('satpy.readers.mirs.retrieve'): + fd.side_effect = fake_coeff_from_fn + loaded_data_arrs = r.load(loadable_ids) + assert loaded_data_arrs + + fd.assert_not_called() From fdba5febe0c7ea13160dd2d94eacb14e748dd039 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Sat, 27 Mar 2021 10:53:07 -0500 Subject: [PATCH 06/29] Remove unused variables and add a noaa-20 test Remove platform name from parameterize in last test for kwargs because they are not used in that test. Add a test for the NOAA-20 platform when the limb_correction assertions are tested and platform name is tested. --- satpy/tests/reader_tests/test_mirs.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/satpy/tests/reader_tests/test_mirs.py b/satpy/tests/reader_tests/test_mirs.py index 0ec614845b..7618c143b7 100644 --- a/satpy/tests/reader_tests/test_mirs.py +++ b/satpy/tests/reader_tests/test_mirs.py @@ -27,6 +27,7 @@ AWIPS_FILE = "IMG_SX.M2.D17037.S1601.E1607.B0000001.WE.HR.ORB.nc" NPP_MIRS_L2_SWATH = "NPR-MIRS-IMG_v11r6_npp_s201702061601000_e201702061607000_c202012201658410.nc" +N20_MIRS_L2_SWATH = "NPR-MIRS-IMG_v11r4_n20_s201702061601000_e201702061607000_c202012201658410.nc" OTHER_MIRS_L2_SWATH = "NPR-MIRS-IMG_v11r4_gpm_s201702061601000_e201702061607000_c202010080001310.nc" EXAMPLE_FILES = [AWIPS_FILE, NPP_MIRS_L2_SWATH, OTHER_MIRS_L2_SWATH] @@ -260,6 +261,7 @@ def _check_attrs(data_arr, platform_name): [ ([AWIPS_FILE], TEST_VARS, "metop-a"), ([NPP_MIRS_L2_SWATH], TEST_VARS, "npp"), + ([N20_MIRS_L2_SWATH], TEST_VARS, "noaa-20"), ([OTHER_MIRS_L2_SWATH], TEST_VARS, "gpm"), ] ) @@ -289,14 +291,15 @@ def test_basic_load(self, filenames, loadable_ids, platform_name): fd.assert_not_called() @pytest.mark.parametrize( - ("filenames", "loadable_ids", "platform_name"), + ("filenames", "loadable_ids"), [ - ([AWIPS_FILE], TEST_VARS, "metop-a"), - ([NPP_MIRS_L2_SWATH], TEST_VARS, "npp"), - ([OTHER_MIRS_L2_SWATH], TEST_VARS, "gpm"), + ([AWIPS_FILE], TEST_VARS), + ([NPP_MIRS_L2_SWATH], TEST_VARS), + ([N20_MIRS_L2_SWATH], TEST_VARS), + ([OTHER_MIRS_L2_SWATH], TEST_VARS), ] ) - def test_kwarg_load(self, filenames, loadable_ids, platform_name): + def test_kwarg_load(self, filenames, loadable_ids): """Test the limb_correction kwarg when filehandler is loaded.""" from satpy.readers import load_reader with mock.patch('satpy.readers.mirs.xr.open_dataset') as od: From a66e3bc63323dff2b5ac237fcb90d9953a27e8fd Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Sun, 28 Mar 2021 09:01:13 -0500 Subject: [PATCH 07/29] Fold reader_kwarg test into basic_load --- satpy/tests/reader_tests/test_mirs.py | 56 ++++++++++----------------- 1 file changed, 20 insertions(+), 36 deletions(-) diff --git a/satpy/tests/reader_tests/test_mirs.py b/satpy/tests/reader_tests/test_mirs.py index 7618c143b7..58f89112a2 100644 --- a/satpy/tests/reader_tests/test_mirs.py +++ b/satpy/tests/reader_tests/test_mirs.py @@ -257,22 +257,28 @@ def _check_attrs(data_arr, platform_name): assert attrs['end_time'] == END_TIME @pytest.mark.parametrize( - ("filenames", "loadable_ids", "platform_name"), + ("filenames", "loadable_ids", "platform_name", "reader_kw"), [ - ([AWIPS_FILE], TEST_VARS, "metop-a"), - ([NPP_MIRS_L2_SWATH], TEST_VARS, "npp"), - ([N20_MIRS_L2_SWATH], TEST_VARS, "noaa-20"), - ([OTHER_MIRS_L2_SWATH], TEST_VARS, "gpm"), + ([AWIPS_FILE], TEST_VARS, "metop-a", None), + ([NPP_MIRS_L2_SWATH], TEST_VARS, "npp", None), + ([N20_MIRS_L2_SWATH], TEST_VARS, "noaa-20", None), + ([OTHER_MIRS_L2_SWATH], TEST_VARS, "gpm", None), + + ([AWIPS_FILE], TEST_VARS, "metop-a", {"limb_correction": False}), + ([NPP_MIRS_L2_SWATH], TEST_VARS, "npp", {"limb_correction": False}), + ([N20_MIRS_L2_SWATH], TEST_VARS, "noaa-20", {"limb_correction": False}), + ([OTHER_MIRS_L2_SWATH], TEST_VARS, "gpm", {"limb_correction": False}), ] ) - def test_basic_load(self, filenames, loadable_ids, platform_name): + def test_basic_load(self, filenames, loadable_ids, + platform_name, reader_kw): """Test that variables are loaded properly.""" from satpy.readers import load_reader with mock.patch('satpy.readers.mirs.xr.open_dataset') as od: od.side_effect = fake_open_dataset r = load_reader(self.reader_configs) loadables = r.select_files_from_pathnames(filenames) - r.create_filehandlers(loadables) + r.create_filehandlers(loadables, fh_kwargs=reader_kw) with mock.patch('satpy.readers.mirs.read_atms_coeff_to_string') as \ fd, mock.patch('satpy.readers.mirs.retrieve'): fd.side_effect = fake_coeff_from_fn @@ -285,33 +291,11 @@ def test_basic_load(self, filenames, loadable_ids, platform_name): self._check_fill(data_arr) self._check_attrs(data_arr, platform_name) - if data_arr.attrs['sensor'] == 'atms': - fd.assert_called() - else: + if reader_kw and not reader_kw['limb_correction']: fd.assert_not_called() - - @pytest.mark.parametrize( - ("filenames", "loadable_ids"), - [ - ([AWIPS_FILE], TEST_VARS), - ([NPP_MIRS_L2_SWATH], TEST_VARS), - ([N20_MIRS_L2_SWATH], TEST_VARS), - ([OTHER_MIRS_L2_SWATH], TEST_VARS), - ] - ) - def test_kwarg_load(self, filenames, loadable_ids): - """Test the limb_correction kwarg when filehandler is loaded.""" - from satpy.readers import load_reader - with mock.patch('satpy.readers.mirs.xr.open_dataset') as od: - od.side_effect = fake_open_dataset - r = load_reader(self.reader_configs) - loadables = r.select_files_from_pathnames(filenames) - r.create_filehandlers(loadables, fh_kwargs={"limb_correction": False}) - - with mock.patch('satpy.readers.mirs.read_atms_coeff_to_string') as \ - fd, mock.patch('satpy.readers.mirs.retrieve'): - fd.side_effect = fake_coeff_from_fn - loaded_data_arrs = r.load(loadable_ids) - assert loaded_data_arrs - - fd.assert_not_called() + else: + sensor = data_arr.attrs['sensor'] + if sensor == 'atms': + fd.assert_called() + else: + fd.assert_not_called() From 6132c67beb6af3eeadf7be7c62cff98111adaea3 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Tue, 30 Mar 2021 07:05:48 -0500 Subject: [PATCH 08/29] Simply if/then statement, split parameterize for readability. --- satpy/tests/reader_tests/test_mirs.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/satpy/tests/reader_tests/test_mirs.py b/satpy/tests/reader_tests/test_mirs.py index 58f89112a2..899c63a098 100644 --- a/satpy/tests/reader_tests/test_mirs.py +++ b/satpy/tests/reader_tests/test_mirs.py @@ -257,19 +257,20 @@ def _check_attrs(data_arr, platform_name): assert attrs['end_time'] == END_TIME @pytest.mark.parametrize( - ("filenames", "loadable_ids", "platform_name", "reader_kw"), + ("filenames", "loadable_ids", "platform_name"), [ - ([AWIPS_FILE], TEST_VARS, "metop-a", None), - ([NPP_MIRS_L2_SWATH], TEST_VARS, "npp", None), - ([N20_MIRS_L2_SWATH], TEST_VARS, "noaa-20", None), - ([OTHER_MIRS_L2_SWATH], TEST_VARS, "gpm", None), - - ([AWIPS_FILE], TEST_VARS, "metop-a", {"limb_correction": False}), - ([NPP_MIRS_L2_SWATH], TEST_VARS, "npp", {"limb_correction": False}), - ([N20_MIRS_L2_SWATH], TEST_VARS, "noaa-20", {"limb_correction": False}), - ([OTHER_MIRS_L2_SWATH], TEST_VARS, "gpm", {"limb_correction": False}), + ([AWIPS_FILE], TEST_VARS, "metop-a"), + ([NPP_MIRS_L2_SWATH], TEST_VARS, "npp"), + ([N20_MIRS_L2_SWATH], TEST_VARS, "noaa-20"), + ([OTHER_MIRS_L2_SWATH], TEST_VARS, "gpm"), + + ([AWIPS_FILE], TEST_VARS, "metop-a"), + ([NPP_MIRS_L2_SWATH], TEST_VARS, "npp"), + ([N20_MIRS_L2_SWATH], TEST_VARS, "noaa-20"), + ([OTHER_MIRS_L2_SWATH], TEST_VARS, "gpm"), ] ) + @pytest.mark.parametrize('reader_kw', [{}, {'limb_correction': False}]) def test_basic_load(self, filenames, loadable_ids, platform_name, reader_kw): """Test that variables are loaded properly.""" @@ -286,6 +287,8 @@ def test_basic_load(self, filenames, loadable_ids, assert loaded_data_arrs for data_id, data_arr in loaded_data_arrs.items(): + sensor = data_arr.attrs['sensor'] + if data_id['name'] not in ['latitude', 'longitude']: self._check_area(data_arr) self._check_fill(data_arr) From f4bdbdab008979f6ae7fa4e76f4ee68f9f8cac2d Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Tue, 30 Mar 2021 07:41:56 -0500 Subject: [PATCH 09/29] Simplify assertion if/then statement for limb_correction. --- satpy/tests/reader_tests/test_mirs.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/satpy/tests/reader_tests/test_mirs.py b/satpy/tests/reader_tests/test_mirs.py index 899c63a098..c88f720b03 100644 --- a/satpy/tests/reader_tests/test_mirs.py +++ b/satpy/tests/reader_tests/test_mirs.py @@ -294,11 +294,8 @@ def test_basic_load(self, filenames, loadable_ids, self._check_fill(data_arr) self._check_attrs(data_arr, platform_name) - if reader_kw and not reader_kw['limb_correction']: - fd.assert_not_called() + sensor = data_arr.attrs['sensor'] + if reader_kw.get('limb_correction', True) and sensor == 'atms': + fd.assert_called() else: - sensor = data_arr.attrs['sensor'] - if sensor == 'atms': - fd.assert_called() - else: - fd.assert_not_called() + fd.assert_not_called() From b0140d1654413b2bfb8587bf548590fa4c4a0b83 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Tue, 30 Mar 2021 12:36:54 -0500 Subject: [PATCH 10/29] Remove extra line getting the name of the sensor --- satpy/tests/reader_tests/test_mirs.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/satpy/tests/reader_tests/test_mirs.py b/satpy/tests/reader_tests/test_mirs.py index c88f720b03..c995b0b46c 100644 --- a/satpy/tests/reader_tests/test_mirs.py +++ b/satpy/tests/reader_tests/test_mirs.py @@ -287,8 +287,6 @@ def test_basic_load(self, filenames, loadable_ids, assert loaded_data_arrs for data_id, data_arr in loaded_data_arrs.items(): - sensor = data_arr.attrs['sensor'] - if data_id['name'] not in ['latitude', 'longitude']: self._check_area(data_arr) self._check_fill(data_arr) From 9f85942bd911ac4c7a328d4f8304192a6973b9e8 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Fri, 30 Apr 2021 06:14:27 -0500 Subject: [PATCH 11/29] Use valid range when present Fixes a bug in which valid range was ignored. --- satpy/readers/mirs.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/satpy/readers/mirs.py b/satpy/readers/mirs.py index d1efb42420..4b19779e36 100644 --- a/satpy/readers/mirs.py +++ b/satpy/readers/mirs.py @@ -356,6 +356,17 @@ def _fill_data(self, data_arr, attrs): data_arr = data_arr.where(data_arr != fill_value, fill_out) return data_arr, attrs + def _get_valid_range(self, data_arr, attrs): + # handle valid_range + valid_range = attrs.pop('valid_range', None) + if isinstance(valid_range, np.ndarray): + valid_min, valid_max = valid_range + + if valid_min is not None and valid_max is not None: + data_arr = data_arr.where((data_arr >= valid_min) & + (data_arr <= valid_max)) + return data_arr, attrs + def get_dataset(self, ds_id, ds_info): """Get datasets.""" if 'dependencies' in ds_info.keys(): @@ -483,6 +494,7 @@ def __getitem__(self, item): attrs = data.attrs.copy() data, attrs = self._scale_data(data, attrs) data, attrs = self._fill_data(data, attrs) + data, attrs = self._get_valid_range(data, attrs) # 'Freq' dimension causes issues in other processing if 'Freq' in data.coords: From d1a0bb50e8951cc84ceb2901ab8f60eb418680a7 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Fri, 30 Apr 2021 06:49:51 -0500 Subject: [PATCH 12/29] Check to confirm that valid range is no longer in attributes. --- satpy/tests/reader_tests/test_mirs.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/satpy/tests/reader_tests/test_mirs.py b/satpy/tests/reader_tests/test_mirs.py index c995b0b46c..bc111fbb6e 100644 --- a/satpy/tests/reader_tests/test_mirs.py +++ b/satpy/tests/reader_tests/test_mirs.py @@ -247,6 +247,11 @@ def _check_fill(data_arr): # we started with float32, it should stay that way assert data_arr.dtype.type == np.float64 + @staticmethod + def _check_valid_range(data_arr): + # valid_range is popped out of data_arr.attrs when it is applied + assert 'valid_range' not in data_arr.attrs + @staticmethod def _check_attrs(data_arr, platform_name): attrs = data_arr.attrs @@ -290,6 +295,7 @@ def test_basic_load(self, filenames, loadable_ids, if data_id['name'] not in ['latitude', 'longitude']: self._check_area(data_arr) self._check_fill(data_arr) + self._check_valid_range(data_arr) self._check_attrs(data_arr, platform_name) sensor = data_arr.attrs['sensor'] From 582837deb0ffd7258a8806c30121723ed684d6fa Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Fri, 30 Apr 2021 13:17:41 -0500 Subject: [PATCH 13/29] Add a test to check valid range was applied correctly. Change get_valid_range=>apply_valid_range and check for case of valid_range not None --- satpy/readers/mirs.py | 6 +++--- satpy/tests/reader_tests/test_mirs.py | 12 ++++++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/satpy/readers/mirs.py b/satpy/readers/mirs.py index 4b19779e36..789ff4f07d 100644 --- a/satpy/readers/mirs.py +++ b/satpy/readers/mirs.py @@ -356,10 +356,10 @@ def _fill_data(self, data_arr, attrs): data_arr = data_arr.where(data_arr != fill_value, fill_out) return data_arr, attrs - def _get_valid_range(self, data_arr, attrs): + def _apply_valid_range(self, data_arr, attrs): # handle valid_range valid_range = attrs.pop('valid_range', None) - if isinstance(valid_range, np.ndarray): + if valid_range is not None: valid_min, valid_max = valid_range if valid_min is not None and valid_max is not None: @@ -494,7 +494,7 @@ def __getitem__(self, item): attrs = data.attrs.copy() data, attrs = self._scale_data(data, attrs) data, attrs = self._fill_data(data, attrs) - data, attrs = self._get_valid_range(data, attrs) + data, attrs = self._apply_valid_range(data, attrs) # 'Freq' dimension causes issues in other processing if 'Freq' in data.coords: diff --git a/satpy/tests/reader_tests/test_mirs.py b/satpy/tests/reader_tests/test_mirs.py index bc111fbb6e..aaf10859d8 100644 --- a/satpy/tests/reader_tests/test_mirs.py +++ b/satpy/tests/reader_tests/test_mirs.py @@ -248,9 +248,17 @@ def _check_fill(data_arr): assert data_arr.dtype.type == np.float64 @staticmethod - def _check_valid_range(data_arr): + def _check_valid_range(filename, data_arr): # valid_range is popped out of data_arr.attrs when it is applied assert 'valid_range' not in data_arr.attrs + var_name = data_arr.attrs["name"] + data = fake_open_dataset(filename) + if "btemp" in var_name: + data = data['BT'] + if "valid_range" in data.attrs: + valid_range = data.attrs["valid_range"] + assert data_arr.data.min() > valid_range[0] + assert data_arr.data.max() < valid_range[1] @staticmethod def _check_attrs(data_arr, platform_name): @@ -295,7 +303,7 @@ def test_basic_load(self, filenames, loadable_ids, if data_id['name'] not in ['latitude', 'longitude']: self._check_area(data_arr) self._check_fill(data_arr) - self._check_valid_range(data_arr) + self._check_valid_range(filenames[0], data_arr) self._check_attrs(data_arr, platform_name) sensor = data_arr.attrs['sensor'] From 879fce4bc1ef9f0192b40564164f49f63dd3e7c5 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Fri, 30 Apr 2021 14:47:58 -0500 Subject: [PATCH 14/29] valid_range is inclusive so include both min/max in acceptable values only read fake_data one time, not every time check_valid_range is called. --- satpy/tests/reader_tests/test_mirs.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/satpy/tests/reader_tests/test_mirs.py b/satpy/tests/reader_tests/test_mirs.py index aaf10859d8..63d48b37fa 100644 --- a/satpy/tests/reader_tests/test_mirs.py +++ b/satpy/tests/reader_tests/test_mirs.py @@ -248,17 +248,11 @@ def _check_fill(data_arr): assert data_arr.dtype.type == np.float64 @staticmethod - def _check_valid_range(filename, data_arr): + def _check_valid_range(data_arr, test_valid_range): # valid_range is popped out of data_arr.attrs when it is applied assert 'valid_range' not in data_arr.attrs - var_name = data_arr.attrs["name"] - data = fake_open_dataset(filename) - if "btemp" in var_name: - data = data['BT'] - if "valid_range" in data.attrs: - valid_range = data.attrs["valid_range"] - assert data_arr.data.min() > valid_range[0] - assert data_arr.data.max() < valid_range[1] + assert data_arr.data.min() >= test_valid_range[0] + assert data_arr.data.max() <= test_valid_range[1] @staticmethod def _check_attrs(data_arr, platform_name): @@ -299,13 +293,20 @@ def test_basic_load(self, filenames, loadable_ids, loaded_data_arrs = r.load(loadable_ids) assert loaded_data_arrs - for data_id, data_arr in loaded_data_arrs.items(): - if data_id['name'] not in ['latitude', 'longitude']: + test_data = fake_open_dataset(filenames[0]) + for _data_id, data_arr in loaded_data_arrs.items(): + var_name = data_arr.attrs["name"] + if var_name not in ['latitude', 'longitude']: self._check_area(data_arr) self._check_fill(data_arr) - self._check_valid_range(filenames[0], data_arr) self._check_attrs(data_arr, platform_name) + input_fake_data = test_data['BT'] if "btemp" in var_name \ + else test_data[var_name] + if "valid_range" in input_fake_data.attrs: + valid_range = input_fake_data.attrs["valid_range"] + self._check_valid_range(data_arr, valid_range) + sensor = data_arr.attrs['sensor'] if reader_kw.get('limb_correction', True) and sensor == 'atms': fd.assert_called() From 08a82ce2bec5aad05558401b513dada6c53abcda Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Thu, 6 May 2021 06:52:58 -0500 Subject: [PATCH 15/29] Add attributes when missing/apply attributes from both file and yaml including _FillValue scale. - Fill value needs to be scaled in these files. - Try to combine yaml and file attributes with file attributes taking precedence over yaml. - Make sure units is saved in data array attributes --- satpy/etc/readers/mirs.yaml | 98 ++++++++++++++++++++++++++++++++++++- satpy/readers/mirs.py | 33 ++++++++++--- 2 files changed, 122 insertions(+), 9 deletions(-) diff --git a/satpy/etc/readers/mirs.yaml b/satpy/etc/readers/mirs.yaml index b968c849c2..6c6a53974a 100644 --- a/satpy/etc/readers/mirs.yaml +++ b/satpy/etc/readers/mirs.yaml @@ -25,4 +25,100 @@ file_types: file_patterns: - 'IMG_SX.{platform_shortname}.D{start_time:%y%j.S%H%M}.E{end_time:%H%M}.B{num}.WE.HR.ORB.nc' -datasets: {} +datasets: + longitude: + name: longitude + file_type: metop_amsu + file_key: Longitude + default_units: degrees + standard_name: longitude + latitude: + name: latitude + file_type: metop_amsu + file_key: Latitude + default_units: degrees + standard_name: latitude + rain_rate: + name: RR + long_name: rain_rate + file_key: RR + file_type: metop_amsu + scale_factor: 0.1 + _FillValue: -99.9 + default_units: mm/hr + coordinates: [longitude, latitude] + mask: + name: Sfc_type + file_key: Sfc_type + file_type: metop_amsu + description: Surface Type:0-ocean,1-sea ice,2-land,3-snow + default_units: 1 + coordinates: [longitude, latitude] + sea_ice: + name: SIce + long_name: sea_ice + file_key: SIce + file_type: metop_amsu + default_units: "%" + coordinates: [longitude, latitude] + snow_cover: + name: Snow + long_name: snow_cover + file_key: Snow + file_type: metop_amsu + default_units: '1' + coordinates: [longitude, latitude] + total_precipitable_water: + name: TPW + long_name: total_precipitable_water + file_key: TPW + file_type: metop_amsu + scale_factor: 0.1 + _FillValue: -99.9 + default_units: mm + coordinates: [longitude, latitude] + swe: + name: SWE + long_name: snow_water_equivalence + file_key: SWE + file_type: metop_amsu + scale_factor: 0.01 + _FillValue: -999s + default_units: cm + coordinates: [longitude, latitude] + cloud_liquid_water: + name: CLW + long_name: Cloud liquid Water + file_key: CLW + file_type: metop_amsu + scale_factor: 0.01 + _FillValue: -999s + default_units: mm + coordinates: [longitude, latitude] + skin_temperature: + name: TSkin + long_name: Skin Temperature + file_key: TSkin + file_type: metop_amsu + scale_factor: 0.01 + _FillValue: -999s + default_units: K + coordinates: [longitude, latitude] + snow_fall_rate: + name: SFR + file_key: SFR + file_type: metop_amsu + long_name: Snow Fall Rate + scale_factor: 0.01 + _FillValue: -999s + default_units: mm/hr + coordinates: [longitude, latitude] + brightness_temperature: + name: BT + file_key: BT + file_type: metop_amsu + long_name: brightness_temperature + scale_factor: 0.01 + _FillValue: -999s + description: Channel Brightness Temperature for every channel + default_units: K diff --git a/satpy/readers/mirs.py b/satpy/readers/mirs.py index 789ff4f07d..7d47fdd70b 100644 --- a/satpy/readers/mirs.py +++ b/satpy/readers/mirs.py @@ -334,23 +334,26 @@ def _nan_for_dtype(data_arr_dtype): return np.nan @staticmethod - def _scale_data(data_arr, attrs): + def _scale_data(data_arr, attrs, scale_factor): # handle scaling # take special care for integer/category fields - scale_factor = attrs.pop('scale_factor', 1.) + add_offset = attrs.pop('add_offset', 0.) scaling_needed = not (scale_factor == 1 and add_offset == 0) if scaling_needed: data_arr = data_arr * scale_factor + add_offset return data_arr, attrs - def _fill_data(self, data_arr, attrs): + def _fill_data(self, data_arr, attrs, scale_factor): + try: global_attr_fill = self.nc.missing_value except AttributeError: global_attr_fill = None fill_value = attrs.pop('_FillValue', global_attr_fill) + fill_value = fill_value * scale_factor + fill_out = self._nan_for_dtype(data_arr.dtype) if fill_value is not None: data_arr = data_arr.where(data_arr != fill_value, fill_out) @@ -386,7 +389,9 @@ def get_dataset(self, ds_id, ds_info): else: data = self[ds_id['name']] - data.attrs = self.get_metadata(ds_info) + data, attrs = self.apply_attributes(data, ds_info) + data.attrs = self.get_metadata(attrs) + return data def _available_if_this_file_type(self, configured_datasets): @@ -482,6 +487,22 @@ def available_datasets(self, configured_datasets=None): yield from self._available_new_datasets() yield from self._available_btemp_datasets() + def apply_attributes(self, data, ds_info): + """Combine attributes from file and yaml and apply. + + File attributes should take precedence over yaml if both are present + + """ + attrs = data.attrs.copy() + # let file attributes take precedence + ds_info.update(attrs) + scale_factor = attrs.pop('scale_factor', 1.) + data, attrs = self._scale_data(data, ds_info, scale_factor) + data, attrs = self._fill_data(data, ds_info, scale_factor) + data, attrs = self._apply_valid_range(data, ds_info) + + return data, ds_info + def __getitem__(self, item): """Wrap around `self.nc[item]`. @@ -491,10 +512,6 @@ def __getitem__(self, item): """ data = self.nc[item] - attrs = data.attrs.copy() - data, attrs = self._scale_data(data, attrs) - data, attrs = self._fill_data(data, attrs) - data, attrs = self._apply_valid_range(data, attrs) # 'Freq' dimension causes issues in other processing if 'Freq' in data.coords: From ad76084971b277c2bf6882651b243ee8da99bf3f Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Tue, 11 May 2021 09:37:27 -0500 Subject: [PATCH 16/29] Fix _FillValue so that it is read as an integer Fix name of units --- satpy/etc/readers/mirs.yaml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/satpy/etc/readers/mirs.yaml b/satpy/etc/readers/mirs.yaml index 6c6a53974a..4e2ee66857 100644 --- a/satpy/etc/readers/mirs.yaml +++ b/satpy/etc/readers/mirs.yaml @@ -30,13 +30,13 @@ datasets: name: longitude file_type: metop_amsu file_key: Longitude - default_units: degrees + units: degrees standard_name: longitude latitude: name: latitude file_type: metop_amsu file_key: Latitude - default_units: degrees + units: degrees standard_name: latitude rain_rate: name: RR @@ -45,28 +45,28 @@ datasets: file_type: metop_amsu scale_factor: 0.1 _FillValue: -99.9 - default_units: mm/hr + units: mm/hr coordinates: [longitude, latitude] mask: name: Sfc_type file_key: Sfc_type file_type: metop_amsu description: Surface Type:0-ocean,1-sea ice,2-land,3-snow - default_units: 1 + units: 1 coordinates: [longitude, latitude] sea_ice: name: SIce long_name: sea_ice file_key: SIce file_type: metop_amsu - default_units: "%" + units: "%" coordinates: [longitude, latitude] snow_cover: name: Snow long_name: snow_cover file_key: Snow file_type: metop_amsu - default_units: '1' + units: '1' coordinates: [longitude, latitude] total_precipitable_water: name: TPW @@ -75,7 +75,7 @@ datasets: file_type: metop_amsu scale_factor: 0.1 _FillValue: -99.9 - default_units: mm + units: mm coordinates: [longitude, latitude] swe: name: SWE @@ -83,8 +83,8 @@ datasets: file_key: SWE file_type: metop_amsu scale_factor: 0.01 - _FillValue: -999s - default_units: cm + _FillValue: -999 + units: cm coordinates: [longitude, latitude] cloud_liquid_water: name: CLW @@ -92,8 +92,8 @@ datasets: file_key: CLW file_type: metop_amsu scale_factor: 0.01 - _FillValue: -999s - default_units: mm + _FillValue: -999 + units: mm coordinates: [longitude, latitude] skin_temperature: name: TSkin @@ -101,8 +101,8 @@ datasets: file_key: TSkin file_type: metop_amsu scale_factor: 0.01 - _FillValue: -999s - default_units: K + _FillValue: -999 + units: K coordinates: [longitude, latitude] snow_fall_rate: name: SFR @@ -110,8 +110,8 @@ datasets: file_type: metop_amsu long_name: Snow Fall Rate scale_factor: 0.01 - _FillValue: -999s - default_units: mm/hr + _FillValue: -999 + units: mm/hr coordinates: [longitude, latitude] brightness_temperature: name: BT @@ -119,6 +119,6 @@ datasets: file_type: metop_amsu long_name: brightness_temperature scale_factor: 0.01 - _FillValue: -999s + _FillValue: -999 description: Channel Brightness Temperature for every channel - default_units: K + units: K From b5b3f862dafdb6d7edaa3c84a2dd70d1e098ad3d Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Tue, 11 May 2021 09:39:10 -0500 Subject: [PATCH 17/29] Update reading of yaml - Need to make sure that yaml metadata when present is used and supplements attributes from data_arr. - Apply scaling to valid_range and fill_value - move location of apply_attributes - change name of get_metadata --- satpy/readers/mirs.py | 88 +++++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 41 deletions(-) diff --git a/satpy/readers/mirs.py b/satpy/readers/mirs.py index 7d47fdd70b..443d988f4a 100644 --- a/satpy/readers/mirs.py +++ b/satpy/readers/mirs.py @@ -309,7 +309,7 @@ def _get_coeff_filenames(self): return coeff_fn - def get_metadata(self, ds_info): + def update_metadata(self, ds_info): """Get metadata.""" metadata = {} metadata.update(ds_info) @@ -334,42 +334,60 @@ def _nan_for_dtype(data_arr_dtype): return np.nan @staticmethod - def _scale_data(data_arr, attrs, scale_factor): - # handle scaling - # take special care for integer/category fields - - add_offset = attrs.pop('add_offset', 0.) + def _scale_data(data_arr, scale_factor, add_offset): + """Scale data, if needed.""" scaling_needed = not (scale_factor == 1 and add_offset == 0) if scaling_needed: data_arr = data_arr * scale_factor + add_offset - return data_arr, attrs - - def _fill_data(self, data_arr, attrs, scale_factor): - - try: - global_attr_fill = self.nc.missing_value - except AttributeError: - global_attr_fill = None - fill_value = attrs.pop('_FillValue', global_attr_fill) - - fill_value = fill_value * scale_factor + return data_arr - fill_out = self._nan_for_dtype(data_arr.dtype) + def _fill_data(self, data_arr, fill_value, scale_factor, add_offset): + """Fill missing data with NaN.""" if fill_value is not None: + fill_value = self._scale_data(fill_value, scale_factor, add_offset) + fill_out = self._nan_for_dtype(data_arr.dtype) data_arr = data_arr.where(data_arr != fill_value, fill_out) - return data_arr, attrs + return data_arr - def _apply_valid_range(self, data_arr, attrs): - # handle valid_range + def _apply_valid_range(self, data_arr, attrs, scale_factor, add_offset): + """Get and apply valid_range.""" valid_range = attrs.pop('valid_range', None) if valid_range is not None: valid_min, valid_max = valid_range + valid_min = self._scale_data(valid_min, scale_factor, add_offset) + valid_max = self._scale_data(valid_max, scale_factor, add_offset) if valid_min is not None and valid_max is not None: data_arr = data_arr.where((data_arr >= valid_min) & (data_arr <= valid_max)) return data_arr, attrs + def apply_attributes(self, data, ds_info): + """Combine attributes from file and yaml and apply. + + File attributes should take precedence over yaml if both are present + + """ + # let file metadata take precedence over ds_info from yaml, + # but if yaml has more to offer, include it here. + ds_info.update(data.attrs) + + try: + global_attr_fill = self.nc.missing_value + except AttributeError: + global_attr_fill = 1.0 + + scale = ds_info.pop('scale_factor', 1.0) + offset = ds_info.pop('add_offset', 0.) + fill_value = ds_info.pop("_FillValue", global_attr_fill) + + data = self._scale_data(data, scale, offset) + data = self._fill_data(data, fill_value, scale, offset) + data, combined_metadata = self._apply_valid_range(data, ds_info, scale, offset) + data.attrs = self.update_metadata(combined_metadata) + + return data + def get_dataset(self, ds_id, ds_info): """Get datasets.""" if 'dependencies' in ds_info.keys(): @@ -389,12 +407,12 @@ def get_dataset(self, ds_id, ds_info): else: data = self[ds_id['name']] - data, attrs = self.apply_attributes(data, ds_info) - data.attrs = self.get_metadata(attrs) + data = self.apply_attributes(data, ds_info) return data def _available_if_this_file_type(self, configured_datasets): + handled_vars = set() for is_avail, ds_info in (configured_datasets or []): if is_avail is not None: # some other file handler said it has this dataset @@ -402,7 +420,10 @@ def _available_if_this_file_type(self, configured_datasets): # file handler so let's yield early yield is_avail, ds_info continue + if self.file_type_matches(ds_info['file_type']): + handled_vars.add(ds_info['name']) yield self.file_type_matches(ds_info['file_type']), ds_info + yield from self._available_new_datasets(handled_vars) def _count_channel_repeat_number(self): """Count channel/polarization pair repetition.""" @@ -462,10 +483,12 @@ def _is_2d_yx_data_array(self, data_arr): has_x_dim = data_arr.dims[1] == "x" return has_y_dim and has_x_dim - def _available_new_datasets(self): + def _available_new_datasets(self, handled_vars): """Metadata for available variables other than BT.""" possible_vars = list(self.nc.items()) + list(self.nc.coords.items()) for var_name, data_arr in possible_vars: + if var_name in handled_vars: + continue if data_arr.ndim != 2: # we don't currently handle non-2D variables continue @@ -484,25 +507,8 @@ def available_datasets(self, configured_datasets=None): """ yield from self._available_if_this_file_type(configured_datasets) - yield from self._available_new_datasets() yield from self._available_btemp_datasets() - def apply_attributes(self, data, ds_info): - """Combine attributes from file and yaml and apply. - - File attributes should take precedence over yaml if both are present - - """ - attrs = data.attrs.copy() - # let file attributes take precedence - ds_info.update(attrs) - scale_factor = attrs.pop('scale_factor', 1.) - data, attrs = self._scale_data(data, ds_info, scale_factor) - data, attrs = self._fill_data(data, ds_info, scale_factor) - data, attrs = self._apply_valid_range(data, ds_info) - - return data, ds_info - def __getitem__(self, item): """Wrap around `self.nc[item]`. From bb9df5a0e808ed03ba663890d26fc01d81a6997b Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Tue, 11 May 2021 12:38:36 -0500 Subject: [PATCH 18/29] Test units for TEST_VARS --- satpy/etc/readers/mirs.yaml | 6 +++--- satpy/readers/mirs.py | 2 +- satpy/tests/reader_tests/test_mirs.py | 17 ++++++++++------- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/satpy/etc/readers/mirs.yaml b/satpy/etc/readers/mirs.yaml index 4e2ee66857..b33f16fa80 100644 --- a/satpy/etc/readers/mirs.yaml +++ b/satpy/etc/readers/mirs.yaml @@ -52,7 +52,7 @@ datasets: file_key: Sfc_type file_type: metop_amsu description: Surface Type:0-ocean,1-sea ice,2-land,3-snow - units: 1 + units: "1" coordinates: [longitude, latitude] sea_ice: name: SIce @@ -102,7 +102,7 @@ datasets: file_type: metop_amsu scale_factor: 0.01 _FillValue: -999 - units: K + units: Kelvin coordinates: [longitude, latitude] snow_fall_rate: name: SFR @@ -121,4 +121,4 @@ datasets: scale_factor: 0.01 _FillValue: -999 description: Channel Brightness Temperature for every channel - units: K + units: Kelvin diff --git a/satpy/readers/mirs.py b/satpy/readers/mirs.py index 443d988f4a..adeabb85af 100644 --- a/satpy/readers/mirs.py +++ b/satpy/readers/mirs.py @@ -458,7 +458,7 @@ def _available_btemp_datasets(self): 'file_type': self.filetype_info['file_type'], 'name': new_name, 'description': desc_bt, - 'units': 'K', + 'units': 'Kelvin', 'channel_index': idx, 'frequency': "{}GHz".format(normal_f), 'polarization': normal_p, diff --git a/satpy/tests/reader_tests/test_mirs.py b/satpy/tests/reader_tests/test_mirs.py index 63d48b37fa..8bc9982c6a 100644 --- a/satpy/tests/reader_tests/test_mirs.py +++ b/satpy/tests/reader_tests/test_mirs.py @@ -25,12 +25,12 @@ import numpy as np import xarray as xr -AWIPS_FILE = "IMG_SX.M2.D17037.S1601.E1607.B0000001.WE.HR.ORB.nc" +METOP_FILE = "IMG_SX.M2.D17037.S1601.E1607.B0000001.WE.HR.ORB.nc" NPP_MIRS_L2_SWATH = "NPR-MIRS-IMG_v11r6_npp_s201702061601000_e201702061607000_c202012201658410.nc" N20_MIRS_L2_SWATH = "NPR-MIRS-IMG_v11r4_n20_s201702061601000_e201702061607000_c202012201658410.nc" OTHER_MIRS_L2_SWATH = "NPR-MIRS-IMG_v11r4_gpm_s201702061601000_e201702061607000_c202010080001310.nc" -EXAMPLE_FILES = [AWIPS_FILE, NPP_MIRS_L2_SWATH, OTHER_MIRS_L2_SWATH] +EXAMPLE_FILES = [METOP_FILE, NPP_MIRS_L2_SWATH, OTHER_MIRS_L2_SWATH] N_CHANNEL = 3 N_FOV = 96 @@ -50,6 +50,8 @@ DS_IDS = ['RR', 'longitude', 'latitude'] TEST_VARS = ['btemp_88v1', 'btemp_88v2', 'btemp_22h', 'RR', 'Sfc_type'] +DEFAULT_UNITS = {'btemp_88v1': 'Kelvin', 'btemp_88v2': 'Kelvin', + 'btemp_22h': 'Kelvin', 'RR': 'mm/hr', 'Sfc_type': "1"} PLATFORM = {"M2": "metop-a", "NPP": "npp", "GPM": "gpm"} SENSOR = {"m2": "amsu-mhs", "npp": "atms", "gpm": "GPI"} @@ -179,7 +181,7 @@ def _get_datasets_with_less_attributes(): def fake_open_dataset(filename, **kwargs): """Create a Dataset similar to reading an actual file with xarray.open_dataset.""" - if filename == AWIPS_FILE: + if filename == METOP_FILE: return _get_datasets_with_less_attributes() return _get_datasets_with_attributes() @@ -197,7 +199,7 @@ def setup_method(self): @pytest.mark.parametrize( ("filenames", "expected_loadables"), [ - ([AWIPS_FILE], 1), + ([METOP_FILE], 1), ([NPP_MIRS_L2_SWATH], 1), ([OTHER_MIRS_L2_SWATH], 1), ] @@ -217,7 +219,7 @@ def test_reader_creation(self, filenames, expected_loadables): @pytest.mark.parametrize( ("filenames", "expected_datasets"), [ - ([AWIPS_FILE], DS_IDS), + ([METOP_FILE], DS_IDS), ([NPP_MIRS_L2_SWATH], DS_IDS), ([OTHER_MIRS_L2_SWATH], DS_IDS), ] @@ -266,12 +268,12 @@ def _check_attrs(data_arr, platform_name): @pytest.mark.parametrize( ("filenames", "loadable_ids", "platform_name"), [ - ([AWIPS_FILE], TEST_VARS, "metop-a"), + ([METOP_FILE], TEST_VARS, "metop-a"), ([NPP_MIRS_L2_SWATH], TEST_VARS, "npp"), ([N20_MIRS_L2_SWATH], TEST_VARS, "noaa-20"), ([OTHER_MIRS_L2_SWATH], TEST_VARS, "gpm"), - ([AWIPS_FILE], TEST_VARS, "metop-a"), + ([METOP_FILE], TEST_VARS, "metop-a"), ([NPP_MIRS_L2_SWATH], TEST_VARS, "npp"), ([N20_MIRS_L2_SWATH], TEST_VARS, "noaa-20"), ([OTHER_MIRS_L2_SWATH], TEST_VARS, "gpm"), @@ -312,3 +314,4 @@ def test_basic_load(self, filenames, loadable_ids, fd.assert_called() else: fd.assert_not_called() + assert data_arr.attrs['units'] == DEFAULT_UNITS[var_name] From f4010fcbb73d0fad47f9d6fb6160c24aa1237330 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Wed, 12 May 2021 13:14:30 -0500 Subject: [PATCH 19/29] Add descriptions for some of the variables in the yaml. --- satpy/etc/readers/mirs.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/satpy/etc/readers/mirs.yaml b/satpy/etc/readers/mirs.yaml index b33f16fa80..2fbc27d974 100644 --- a/satpy/etc/readers/mirs.yaml +++ b/satpy/etc/readers/mirs.yaml @@ -40,6 +40,7 @@ datasets: standard_name: latitude rain_rate: name: RR + description: Rain Rate long_name: rain_rate file_key: RR file_type: metop_amsu @@ -49,13 +50,14 @@ datasets: coordinates: [longitude, latitude] mask: name: Sfc_type - file_key: Sfc_type - file_type: metop_amsu +type of surface:0-ocean,1-sea ice,2-land,3-snow") file_key: Sfc_type + file_type: me"Channel Brightness Temperature for every channel", units="K")top_amsu description: Surface Type:0-ocean,1-sea ice,2-land,3-snow units: "1" coordinates: [longitude, latitude] sea_ice: name: SIce + description: Sea Ice long_name: sea_ice file_key: SIce file_type: metop_amsu @@ -63,6 +65,7 @@ datasets: coordinates: [longitude, latitude] snow_cover: name: Snow + description: Snow Cover long_name: snow_cover file_key: Snow file_type: metop_amsu @@ -70,6 +73,7 @@ datasets: coordinates: [longitude, latitude] total_precipitable_water: name: TPW + description: Total Precipitable Water long_name: total_precipitable_water file_key: TPW file_type: metop_amsu @@ -79,6 +83,7 @@ datasets: coordinates: [longitude, latitude] swe: name: SWE + description: Snow Water Equivalence long_name: snow_water_equivalence file_key: SWE file_type: metop_amsu @@ -88,6 +93,7 @@ datasets: coordinates: [longitude, latitude] cloud_liquid_water: name: CLW + description: Cloud Liquid Water long_name: Cloud liquid Water file_key: CLW file_type: metop_amsu @@ -97,6 +103,7 @@ datasets: coordinates: [longitude, latitude] skin_temperature: name: TSkin + description: skin temperature long_name: Skin Temperature file_key: TSkin file_type: metop_amsu @@ -106,6 +113,7 @@ datasets: coordinates: [longitude, latitude] snow_fall_rate: name: SFR + description: snow fall rate file_key: SFR file_type: metop_amsu long_name: Snow Fall Rate From 6988f3b22ae8b6266e229c44a7b8487d1b4a34bb Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Thu, 13 May 2021 07:49:17 -0500 Subject: [PATCH 20/29] BUG FIXES: apply attributes before limb correction and fix typos in yaml - apply the attributes before the limb correction so that scaling, valid_range, fill are applied correctly - the yaml had some copy and pasted errors that would make it useless and unreadable - remove BT from yaml and handle attributes in reader - use the yaml for lat/lon attributes from every file --- satpy/etc/readers/mirs.yaml | 31 ++++++++++++------------ satpy/readers/mirs.py | 34 +++++++++++++++------------ satpy/tests/reader_tests/test_mirs.py | 21 ++++++++++------- 3 files changed, 47 insertions(+), 39 deletions(-) diff --git a/satpy/etc/readers/mirs.yaml b/satpy/etc/readers/mirs.yaml index 2fbc27d974..3fea639adb 100644 --- a/satpy/etc/readers/mirs.yaml +++ b/satpy/etc/readers/mirs.yaml @@ -20,6 +20,10 @@ file_types: file_reader: !!python/name:satpy.readers.mirs.MiRSL2ncHandler file_patterns: - 'NPR-MIRS-IMG_v{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S}{extra_num1}_e{end_time:%Y%m%d%H%M%S}{extra_num2}_c{creation_time:%Y%m%d%H%M%S}{extra_num3}.nc' + mirs_gpm: + file_reader: !!python/name:satpy.readers.mirs.MiRSL2ncHandler + file_patterns: + - 'NPR-MIRS-IMG_v{version}_{platform_shortname:gpm}_s{start_time:%Y%m%d%H%M%S}{extra_num1}_e{end_time:%Y%m%d%H%M%S}{extra_num2}_c{creation_time:%Y%m%d%H%M%S}{extra_num3}.nc' metop_amsu: file_reader: !!python/name:satpy.readers.mirs.MiRSL2ncHandler file_patterns: @@ -28,14 +32,20 @@ file_types: datasets: longitude: name: longitude - file_type: metop_amsu + long_name: Longitude of the view (-180,180) + file_type: [ metop_amsu, mirs_atms, mirs_gpm ] file_key: Longitude units: degrees + _FillValue: -999. + valid_range: [ -180., 180. ] standard_name: longitude latitude: name: latitude - file_type: metop_amsu + long_name: Latitude of the view (-90,90) + file_type: [ metop_amsu, mirs_atms, mirs_gpm ] file_key: Latitude + _FillValue: -999. + valid_range: [-90., 90.] units: degrees standard_name: latitude rain_rate: @@ -50,8 +60,8 @@ datasets: coordinates: [longitude, latitude] mask: name: Sfc_type -type of surface:0-ocean,1-sea ice,2-land,3-snow") file_key: Sfc_type - file_type: me"Channel Brightness Temperature for every channel", units="K")top_amsu + file_key: Sfc_type + file_type: metop_amsu description: Surface Type:0-ocean,1-sea ice,2-land,3-snow units: "1" coordinates: [longitude, latitude] @@ -109,7 +119,7 @@ type of surface:0-ocean,1-sea ice,2-land,3-snow") file_key: Sfc_type file_type: metop_amsu scale_factor: 0.01 _FillValue: -999 - units: Kelvin + units: K coordinates: [longitude, latitude] snow_fall_rate: name: SFR @@ -120,13 +130,4 @@ type of surface:0-ocean,1-sea ice,2-land,3-snow") file_key: Sfc_type scale_factor: 0.01 _FillValue: -999 units: mm/hr - coordinates: [longitude, latitude] - brightness_temperature: - name: BT - file_key: BT - file_type: metop_amsu - long_name: brightness_temperature - scale_factor: 0.01 - _FillValue: -999 - description: Channel Brightness Temperature for every channel - units: Kelvin + coordinates: [longitude, latitude] \ No newline at end of file diff --git a/satpy/readers/mirs.py b/satpy/readers/mirs.py index adeabb85af..915ad40fad 100644 --- a/satpy/readers/mirs.py +++ b/satpy/readers/mirs.py @@ -349,9 +349,8 @@ def _fill_data(self, data_arr, fill_value, scale_factor, add_offset): data_arr = data_arr.where(data_arr != fill_value, fill_out) return data_arr - def _apply_valid_range(self, data_arr, attrs, scale_factor, add_offset): + def _apply_valid_range(self, data_arr, valid_range, scale_factor, add_offset): """Get and apply valid_range.""" - valid_range = attrs.pop('valid_range', None) if valid_range is not None: valid_min, valid_max = valid_range valid_min = self._scale_data(valid_min, scale_factor, add_offset) @@ -360,7 +359,7 @@ def _apply_valid_range(self, data_arr, attrs, scale_factor, add_offset): if valid_min is not None and valid_max is not None: data_arr = data_arr.where((data_arr >= valid_min) & (data_arr <= valid_max)) - return data_arr, attrs + return data_arr def apply_attributes(self, data, ds_info): """Combine attributes from file and yaml and apply. @@ -368,31 +367,37 @@ def apply_attributes(self, data, ds_info): File attributes should take precedence over yaml if both are present """ - # let file metadata take precedence over ds_info from yaml, - # but if yaml has more to offer, include it here. - ds_info.update(data.attrs) - try: global_attr_fill = self.nc.missing_value except AttributeError: global_attr_fill = 1.0 + # let file metadata take precedence over ds_info from yaml, + # but if yaml has more to offer, include it here, but fix + # units. + ds_info.update(data.attrs) + scale = ds_info.pop('scale_factor', 1.0) offset = ds_info.pop('add_offset', 0.) fill_value = ds_info.pop("_FillValue", global_attr_fill) + valid_range = ds_info.pop('valid_range', None) + + units_convert = {"Kelvin": "K"} + data_unit = ds_info['units'] + ds_info['units'] = units_convert.get(data_unit, data_unit) data = self._scale_data(data, scale, offset) data = self._fill_data(data, fill_value, scale, offset) - data, combined_metadata = self._apply_valid_range(data, ds_info, scale, offset) - data.attrs = self.update_metadata(combined_metadata) + data = self._apply_valid_range(data, valid_range, scale, offset) - return data + return data, ds_info def get_dataset(self, ds_id, ds_info): """Get datasets.""" if 'dependencies' in ds_info.keys(): idx = ds_info['channel_index'] data = self['BT'] + data, ds_info = self.apply_attributes(data, ds_info) data = data.rename(new_name_or_name_dict=ds_info["name"]) if self.sensor.lower() == "atms" and self.limb_correction: @@ -406,8 +411,9 @@ def get_dataset(self, ds_id, ds_info): data = data[:, :, idx] else: data = self[ds_id['name']] + data, ds_info = self.apply_attributes(data, ds_info) - data = self.apply_attributes(data, ds_info) + data.attrs = self.update_metadata(ds_info) return data @@ -458,7 +464,8 @@ def _available_btemp_datasets(self): 'file_type': self.filetype_info['file_type'], 'name': new_name, 'description': desc_bt, - 'units': 'Kelvin', + 'units': 'K', + 'scale_factor': self.nc['BT'].attrs['scale_factor'], 'channel_index': idx, 'frequency': "{}GHz".format(normal_f), 'polarization': normal_p, @@ -473,9 +480,6 @@ def _get_ds_info_for_data_arr(self, var_name): 'name': var_name, 'coordinates': ["longitude", "latitude"] } - - if var_name in ["longitude", "latitude"]: - ds_info['standard_name'] = var_name return ds_info def _is_2d_yx_data_array(self, data_arr): diff --git a/satpy/tests/reader_tests/test_mirs.py b/satpy/tests/reader_tests/test_mirs.py index 8bc9982c6a..df445ab4e7 100644 --- a/satpy/tests/reader_tests/test_mirs.py +++ b/satpy/tests/reader_tests/test_mirs.py @@ -50,8 +50,8 @@ DS_IDS = ['RR', 'longitude', 'latitude'] TEST_VARS = ['btemp_88v1', 'btemp_88v2', 'btemp_22h', 'RR', 'Sfc_type'] -DEFAULT_UNITS = {'btemp_88v1': 'Kelvin', 'btemp_88v2': 'Kelvin', - 'btemp_22h': 'Kelvin', 'RR': 'mm/hr', 'Sfc_type': "1"} +DEFAULT_UNITS = {'btemp_88v1': 'K', 'btemp_88v2': 'K', + 'btemp_22h': 'K', 'RR': 'mm/hr', 'Sfc_type': "1"} PLATFORM = {"M2": "metop-a", "NPP": "npp", "GPM": "gpm"} SENSOR = {"m2": "amsu-mhs", "npp": "atms", "gpm": "GPI"} @@ -134,7 +134,7 @@ def _get_datasets_with_attributes(**kwargs): attrs = {'missing_value': -999.} ds = xr.Dataset(ds_vars, attrs=attrs) - + ds = ds.assign_coords({"Freq": FREQ, "Latitude": latitude, "Longitude": longitude}) return ds @@ -175,7 +175,7 @@ def _get_datasets_with_less_attributes(): attrs = {'missing_value': -999.} ds = xr.Dataset(ds_vars, attrs=attrs) - + ds = ds.assign_coords({"Freq": FREQ, "Latitude": latitude, "Longitude": longitude}) return ds @@ -256,6 +256,11 @@ def _check_valid_range(data_arr, test_valid_range): assert data_arr.data.min() >= test_valid_range[0] assert data_arr.data.max() <= test_valid_range[1] + @staticmethod + def _check_fill_value(data_arr, test_fill_value): + assert '_FillValue' not in data_arr.attrs + assert test_fill_value not in data_arr.data + @staticmethod def _check_attrs(data_arr, platform_name): attrs = data_arr.attrs @@ -272,11 +277,6 @@ def _check_attrs(data_arr, platform_name): ([NPP_MIRS_L2_SWATH], TEST_VARS, "npp"), ([N20_MIRS_L2_SWATH], TEST_VARS, "noaa-20"), ([OTHER_MIRS_L2_SWATH], TEST_VARS, "gpm"), - - ([METOP_FILE], TEST_VARS, "metop-a"), - ([NPP_MIRS_L2_SWATH], TEST_VARS, "npp"), - ([N20_MIRS_L2_SWATH], TEST_VARS, "noaa-20"), - ([OTHER_MIRS_L2_SWATH], TEST_VARS, "gpm"), ] ) @pytest.mark.parametrize('reader_kw', [{}, {'limb_correction': False}]) @@ -308,6 +308,9 @@ def test_basic_load(self, filenames, loadable_ids, if "valid_range" in input_fake_data.attrs: valid_range = input_fake_data.attrs["valid_range"] self._check_valid_range(data_arr, valid_range) + if "_FillValue" in input_fake_data.attrs: + fill_value = input_fake_data.attrs["_FillValue"] + self._check_fill_value(data_arr, fill_value) sensor = data_arr.attrs['sensor'] if reader_kw.get('limb_correction', True) and sensor == 'atms': From c40ee15ccb3532315be9d0d7168842b61de134f8 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Thu, 13 May 2021 07:58:49 -0500 Subject: [PATCH 21/29] Remove the change to the file_patterns, the extra file_type is not needed for mirs_gpm Added because I did not consider that the limb correction is applied based on sensor which is assigned by platform. The mirs_gpm addition was not needed so I removed it. --- satpy/etc/readers/mirs.yaml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/satpy/etc/readers/mirs.yaml b/satpy/etc/readers/mirs.yaml index 3fea639adb..731524b8ab 100644 --- a/satpy/etc/readers/mirs.yaml +++ b/satpy/etc/readers/mirs.yaml @@ -20,10 +20,6 @@ file_types: file_reader: !!python/name:satpy.readers.mirs.MiRSL2ncHandler file_patterns: - 'NPR-MIRS-IMG_v{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S}{extra_num1}_e{end_time:%Y%m%d%H%M%S}{extra_num2}_c{creation_time:%Y%m%d%H%M%S}{extra_num3}.nc' - mirs_gpm: - file_reader: !!python/name:satpy.readers.mirs.MiRSL2ncHandler - file_patterns: - - 'NPR-MIRS-IMG_v{version}_{platform_shortname:gpm}_s{start_time:%Y%m%d%H%M%S}{extra_num1}_e{end_time:%Y%m%d%H%M%S}{extra_num2}_c{creation_time:%Y%m%d%H%M%S}{extra_num3}.nc' metop_amsu: file_reader: !!python/name:satpy.readers.mirs.MiRSL2ncHandler file_patterns: @@ -33,7 +29,7 @@ datasets: longitude: name: longitude long_name: Longitude of the view (-180,180) - file_type: [ metop_amsu, mirs_atms, mirs_gpm ] + file_type: [ metop_amsu, mirs_atms ] file_key: Longitude units: degrees _FillValue: -999. @@ -42,7 +38,7 @@ datasets: latitude: name: latitude long_name: Latitude of the view (-90,90) - file_type: [ metop_amsu, mirs_atms, mirs_gpm ] + file_type: [ metop_amsu, mirs_atms ] file_key: Latitude _FillValue: -999. valid_range: [-90., 90.] From 3e367c828991a9c786a085f328cdc108b8fc9342 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Thu, 13 May 2021 19:25:40 -0500 Subject: [PATCH 22/29] Add BT to yaml and use in creation of ds_info for data_id --- satpy/readers/mirs.py | 42 +++++++++++++++++------------------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/satpy/readers/mirs.py b/satpy/readers/mirs.py index 915ad40fad..faf93a9a2a 100644 --- a/satpy/readers/mirs.py +++ b/satpy/readers/mirs.py @@ -397,14 +397,15 @@ def get_dataset(self, ds_id, ds_info): if 'dependencies' in ds_info.keys(): idx = ds_info['channel_index'] data = self['BT'] - data, ds_info = self.apply_attributes(data, ds_info) data = data.rename(new_name_or_name_dict=ds_info["name"]) + data, ds_info = self.apply_attributes(data, ds_info) if self.sensor.lower() == "atms" and self.limb_correction: sfc_type_mask = self['Sfc_type'] data = limb_correct_atms_bt(data, sfc_type_mask, self._get_coeff_filenames, ds_info) + self.nc = self.nc.merge(data) else: LOG.info("No Limb Correction applied.") @@ -417,7 +418,13 @@ def get_dataset(self, ds_id, ds_info): return data - def _available_if_this_file_type(self, configured_datasets): + def available_datasets(self, configured_datasets=None): + """Dynamically discover what variables can be loaded from this file. + + See :meth:`satpy.readers.file_handlers.BaseHandler.available_datasets` + for more information. + + """ handled_vars = set() for is_avail, ds_info in (configured_datasets or []): if is_avail is not None: @@ -428,6 +435,8 @@ def _available_if_this_file_type(self, configured_datasets): continue if self.file_type_matches(ds_info['file_type']): handled_vars.add(ds_info['name']) + if ds_info['name'] == 'BT': + yield from self._available_btemp_datasets(ds_info) yield self.file_type_matches(ds_info['file_type']), ds_info yield from self._available_new_datasets(handled_vars) @@ -446,7 +455,7 @@ def _count_channel_repeat_number(self): return chn_total, normals - def _available_btemp_datasets(self): + def _available_btemp_datasets(self, yaml_info): """Create metadata for channel BTs.""" chn_total, normals = self._count_channel_repeat_number() # keep track of current channel count for string description @@ -460,18 +469,17 @@ def _available_btemp_datasets(self): desc_bt = "Channel {} Brightness Temperature at {}GHz {}{}" desc_bt = desc_bt.format(idx, normal_f, normal_p, p_count) - ds_info = { + ds_info = yaml_info.copy() + ds_info.update({ 'file_type': self.filetype_info['file_type'], 'name': new_name, 'description': desc_bt, - 'units': 'K', - 'scale_factor': self.nc['BT'].attrs['scale_factor'], 'channel_index': idx, 'frequency': "{}GHz".format(normal_f), 'polarization': normal_p, 'dependencies': ('BT', 'Sfc_type'), - 'coordinates': ["longitude", "latitude"] - } + 'coordinates': ['longitude', 'latitude'] + }) yield True, ds_info def _get_ds_info_for_data_arr(self, var_name): @@ -503,24 +511,8 @@ def _available_new_datasets(self, handled_vars): ds_info = self._get_ds_info_for_data_arr(var_name) yield True, ds_info - def available_datasets(self, configured_datasets=None): - """Dynamically discover what variables can be loaded from this file. - - See :meth:`satpy.readers.file_handlers.BaseHandler.available_datasets` - for more information. - - """ - yield from self._available_if_this_file_type(configured_datasets) - yield from self._available_btemp_datasets() - def __getitem__(self, item): - """Wrap around `self.nc[item]`. - - Some datasets use a 32-bit float scaling factor like the 'x' and 'y' - variables which causes inaccurate unscaled data values. This method - forces the scale factor to a 64-bit float first. - - """ + """Wrap around `self.nc[item]`.""" data = self.nc[item] # 'Freq' dimension causes issues in other processing From 18d3356c0734a5419631ebee788194fcabdba28d Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Thu, 13 May 2021 19:30:44 -0500 Subject: [PATCH 23/29] commit the yaml mentioned in previous commit --- satpy/etc/readers/mirs.yaml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/satpy/etc/readers/mirs.yaml b/satpy/etc/readers/mirs.yaml index 731524b8ab..d64a3b57a5 100644 --- a/satpy/etc/readers/mirs.yaml +++ b/satpy/etc/readers/mirs.yaml @@ -126,4 +126,15 @@ datasets: scale_factor: 0.01 _FillValue: -999 units: mm/hr - coordinates: [longitude, latitude] \ No newline at end of file + coordinates: [longitude, latitude] + bt: + name: BT + file_key: BT + file_type: [ metop_amsu, mirs_atms ] + description: Channel Brightness Temperature for every channel + long_name: Channel Temperature (K) + units: K + scale_factor: 0.01 + _FillValue: -999 + valid_range: [0, 50000] + standard_name: brightness_temperature From 2ad0ba3b1bc44dd16f06a82412c4a9d8adde42ea Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Thu, 13 May 2021 20:28:48 -0500 Subject: [PATCH 24/29] Take file_key out of BT dataset so that it does not get carried through to btemp dataset information --- satpy/etc/readers/mirs.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/satpy/etc/readers/mirs.yaml b/satpy/etc/readers/mirs.yaml index d64a3b57a5..78e5d330e1 100644 --- a/satpy/etc/readers/mirs.yaml +++ b/satpy/etc/readers/mirs.yaml @@ -129,7 +129,6 @@ datasets: coordinates: [longitude, latitude] bt: name: BT - file_key: BT file_type: [ metop_amsu, mirs_atms ] description: Channel Brightness Temperature for every channel long_name: Channel Temperature (K) From ebc2eff6c4e512e1d8505ccba28daf2734cd31dd Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Fri, 14 May 2021 07:25:21 -0500 Subject: [PATCH 25/29] Don't add more in yaml than necessary If the information is in the metadata of the file, use that, otherwise define in the yaml. Fix getting keys from ds_info in apply_attributes function so it does not needlessly break data that has no units. --- satpy/etc/readers/mirs.yaml | 31 +++------------------------- satpy/readers/mirs.py | 41 ++++++++++++++++++++++--------------- 2 files changed, 28 insertions(+), 44 deletions(-) diff --git a/satpy/etc/readers/mirs.yaml b/satpy/etc/readers/mirs.yaml index 78e5d330e1..558f8a2254 100644 --- a/satpy/etc/readers/mirs.yaml +++ b/satpy/etc/readers/mirs.yaml @@ -28,30 +28,23 @@ file_types: datasets: longitude: name: longitude - long_name: Longitude of the view (-180,180) - file_type: [ metop_amsu, mirs_atms ] + file_type: metop_amsu file_key: Longitude units: degrees - _FillValue: -999. valid_range: [ -180., 180. ] standard_name: longitude latitude: name: latitude - long_name: Latitude of the view (-90,90) - file_type: [ metop_amsu, mirs_atms ] + file_type: metop_amsu file_key: Latitude - _FillValue: -999. valid_range: [-90., 90.] units: degrees standard_name: latitude rain_rate: name: RR description: Rain Rate - long_name: rain_rate file_key: RR file_type: metop_amsu - scale_factor: 0.1 - _FillValue: -99.9 units: mm/hr coordinates: [longitude, latitude] mask: @@ -64,7 +57,6 @@ datasets: sea_ice: name: SIce description: Sea Ice - long_name: sea_ice file_key: SIce file_type: metop_amsu units: "%" @@ -80,41 +72,29 @@ datasets: total_precipitable_water: name: TPW description: Total Precipitable Water - long_name: total_precipitable_water file_key: TPW file_type: metop_amsu - scale_factor: 0.1 - _FillValue: -99.9 units: mm coordinates: [longitude, latitude] swe: name: SWE description: Snow Water Equivalence - long_name: snow_water_equivalence file_key: SWE file_type: metop_amsu - scale_factor: 0.01 - _FillValue: -999 units: cm coordinates: [longitude, latitude] cloud_liquid_water: name: CLW description: Cloud Liquid Water - long_name: Cloud liquid Water file_key: CLW file_type: metop_amsu - scale_factor: 0.01 - _FillValue: -999 units: mm coordinates: [longitude, latitude] skin_temperature: name: TSkin description: skin temperature - long_name: Skin Temperature file_key: TSkin file_type: metop_amsu - scale_factor: 0.01 - _FillValue: -999 units: K coordinates: [longitude, latitude] snow_fall_rate: @@ -122,18 +102,13 @@ datasets: description: snow fall rate file_key: SFR file_type: metop_amsu - long_name: Snow Fall Rate - scale_factor: 0.01 - _FillValue: -999 units: mm/hr coordinates: [longitude, latitude] bt: name: BT - file_type: [ metop_amsu, mirs_atms ] + file_type: metop_amsu description: Channel Brightness Temperature for every channel long_name: Channel Temperature (K) units: K - scale_factor: 0.01 - _FillValue: -999 valid_range: [0, 50000] standard_name: brightness_temperature diff --git a/satpy/readers/mirs.py b/satpy/readers/mirs.py index faf93a9a2a..dd781cf908 100644 --- a/satpy/readers/mirs.py +++ b/satpy/readers/mirs.py @@ -377,15 +377,22 @@ def apply_attributes(self, data, ds_info): # units. ds_info.update(data.attrs) + # special cases + if ds_info['name'] in ["latitude", "longitude"]: + ds_info["standard_name"] = ds_info.get("standard_name", + ds_info['name']) + + # try to assign appropriate units (if "Kelvin" covert to K) + units_convert = {"Kelvin": "K"} + data_unit = ds_info.get('units', None) + if data_unit is not None: + ds_info['units'] = units_convert.get(data_unit, data_unit) + scale = ds_info.pop('scale_factor', 1.0) offset = ds_info.pop('add_offset', 0.) fill_value = ds_info.pop("_FillValue", global_attr_fill) valid_range = ds_info.pop('valid_range', None) - units_convert = {"Kelvin": "K"} - data_unit = ds_info['units'] - ds_info['units'] = units_convert.get(data_unit, data_unit) - data = self._scale_data(data, scale, offset) data = self._fill_data(data, fill_value, scale, offset) data = self._apply_valid_range(data, valid_range, scale, offset) @@ -418,13 +425,8 @@ def get_dataset(self, ds_id, ds_info): return data - def available_datasets(self, configured_datasets=None): - """Dynamically discover what variables can be loaded from this file. - - See :meth:`satpy.readers.file_handlers.BaseHandler.available_datasets` - for more information. + def _available_if_this_type(self, configured_datasets): - """ handled_vars = set() for is_avail, ds_info in (configured_datasets or []): if is_avail is not None: @@ -435,8 +437,6 @@ def available_datasets(self, configured_datasets=None): continue if self.file_type_matches(ds_info['file_type']): handled_vars.add(ds_info['name']) - if ds_info['name'] == 'BT': - yield from self._available_btemp_datasets(ds_info) yield self.file_type_matches(ds_info['file_type']), ds_info yield from self._available_new_datasets(handled_vars) @@ -455,7 +455,7 @@ def _count_channel_repeat_number(self): return chn_total, normals - def _available_btemp_datasets(self, yaml_info): + def _available_btemp_datasets(self): """Create metadata for channel BTs.""" chn_total, normals = self._count_channel_repeat_number() # keep track of current channel count for string description @@ -469,8 +469,7 @@ def _available_btemp_datasets(self, yaml_info): desc_bt = "Channel {} Brightness Temperature at {}GHz {}{}" desc_bt = desc_bt.format(idx, normal_f, normal_p, p_count) - ds_info = yaml_info.copy() - ds_info.update({ + ds_info = { 'file_type': self.filetype_info['file_type'], 'name': new_name, 'description': desc_bt, @@ -479,7 +478,7 @@ def _available_btemp_datasets(self, yaml_info): 'polarization': normal_p, 'dependencies': ('BT', 'Sfc_type'), 'coordinates': ['longitude', 'latitude'] - }) + } yield True, ds_info def _get_ds_info_for_data_arr(self, var_name): @@ -511,6 +510,16 @@ def _available_new_datasets(self, handled_vars): ds_info = self._get_ds_info_for_data_arr(var_name) yield True, ds_info + def available_datasets(self, configured_datasets=None): + """Dynamically discover what variables can be loaded from this file. + + See :meth:`satpy.readers.file_handlers.BaseHandler.available_datasets` + for more information. + + """ + yield from self._available_if_this_type(configured_datasets) + yield from self._available_btemp_datasets() + def __getitem__(self, item): """Wrap around `self.nc[item]`.""" data = self.nc[item] From fa82afa792071e06c55e9de9349acd9f58fa8a79 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Fri, 14 May 2021 08:49:03 -0500 Subject: [PATCH 26/29] Make sure btemp information is only initializing with yaml when necessary. --- satpy/readers/mirs.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/satpy/readers/mirs.py b/satpy/readers/mirs.py index dd781cf908..6050b86b92 100644 --- a/satpy/readers/mirs.py +++ b/satpy/readers/mirs.py @@ -385,8 +385,7 @@ def apply_attributes(self, data, ds_info): # try to assign appropriate units (if "Kelvin" covert to K) units_convert = {"Kelvin": "K"} data_unit = ds_info.get('units', None) - if data_unit is not None: - ds_info['units'] = units_convert.get(data_unit, data_unit) + ds_info['units'] = units_convert.get(data_unit, data_unit) scale = ds_info.pop('scale_factor', 1.0) offset = ds_info.pop('add_offset', 0.) @@ -397,6 +396,8 @@ def apply_attributes(self, data, ds_info): data = self._fill_data(data, fill_value, scale, offset) data = self._apply_valid_range(data, valid_range, scale, offset) + data.attrs = ds_info + return data, ds_info def get_dataset(self, ds_id, ds_info): @@ -425,8 +426,13 @@ def get_dataset(self, ds_id, ds_info): return data - def _available_if_this_type(self, configured_datasets): + def available_datasets(self, configured_datasets=None): + """Dynamically discover what variables can be loaded from this file. + + See :meth:`satpy.readers.file_handlers.BaseHandler.available_datasets` + for more information. + """ handled_vars = set() for is_avail, ds_info in (configured_datasets or []): if is_avail is not None: @@ -435,8 +441,13 @@ def _available_if_this_type(self, configured_datasets): # file handler so let's yield early yield is_avail, ds_info continue + if self.file_type_matches(ds_info['file_type']): handled_vars.add(ds_info['name']) + if ds_info['name'] == 'BT': + yaml_info = ds_info if self.file_type_matches( + ds_info['file_type']) else {} + yield from self._available_btemp_datasets(yaml_info) yield self.file_type_matches(ds_info['file_type']), ds_info yield from self._available_new_datasets(handled_vars) @@ -455,7 +466,7 @@ def _count_channel_repeat_number(self): return chn_total, normals - def _available_btemp_datasets(self): + def _available_btemp_datasets(self, yaml_info): """Create metadata for channel BTs.""" chn_total, normals = self._count_channel_repeat_number() # keep track of current channel count for string description @@ -469,7 +480,8 @@ def _available_btemp_datasets(self): desc_bt = "Channel {} Brightness Temperature at {}GHz {}{}" desc_bt = desc_bt.format(idx, normal_f, normal_p, p_count) - ds_info = { + ds_info = yaml_info.copy() + ds_info.update({ 'file_type': self.filetype_info['file_type'], 'name': new_name, 'description': desc_bt, @@ -478,7 +490,7 @@ def _available_btemp_datasets(self): 'polarization': normal_p, 'dependencies': ('BT', 'Sfc_type'), 'coordinates': ['longitude', 'latitude'] - } + }) yield True, ds_info def _get_ds_info_for_data_arr(self, var_name): @@ -510,16 +522,6 @@ def _available_new_datasets(self, handled_vars): ds_info = self._get_ds_info_for_data_arr(var_name) yield True, ds_info - def available_datasets(self, configured_datasets=None): - """Dynamically discover what variables can be loaded from this file. - - See :meth:`satpy.readers.file_handlers.BaseHandler.available_datasets` - for more information. - - """ - yield from self._available_if_this_type(configured_datasets) - yield from self._available_btemp_datasets() - def __getitem__(self, item): """Wrap around `self.nc[item]`.""" data = self.nc[item] From 6a5c817cfb258e9b75b99c143fd0fc0a3b058486 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Sat, 15 May 2021 06:16:34 -0500 Subject: [PATCH 27/29] Simplify the reading of coefficients so reading old/versus new coefficients is consistent Old coefficients have a blank line at end, new coefficients do not. Previously, n_chn and n_fov were fixed in the coefficient reading. So, return to that so it is easier to swap new and old coefficients if necessary for testing. --- satpy/readers/mirs.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/satpy/readers/mirs.py b/satpy/readers/mirs.py index 6050b86b92..98fde9e81b 100644 --- a/satpy/readers/mirs.py +++ b/satpy/readers/mirs.py @@ -93,11 +93,8 @@ def read_atms_coeff_to_string(fn): def read_atms_limb_correction_coefficients(fn): """Read the limb correction files.""" coeff_str = read_atms_coeff_to_string(fn) - # there should be 22 channels and 96 fov in the coefficient file. (read last line and get values) - n_chn = (coeff_str[-1].replace(" ", " ").split(" ")[1]) - n_fov = (coeff_str[-1].replace(" ", " ").split(" ")[2]) - n_chn = int(n_chn) - n_fov = int(n_fov) + n_chn = 22 + n_fov = 96 if n_chn < 22: LOG.warning('Coefficient file has less than 22 channels: %s' % n_chn) if n_fov < 96: From 1b79da92c0f2b025ee85960da449433302b4e064 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Sat, 15 May 2021 09:39:00 -0500 Subject: [PATCH 28/29] Take out check for n_chn and n_fov since they are fixed now. --- satpy/readers/mirs.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/satpy/readers/mirs.py b/satpy/readers/mirs.py index 98fde9e81b..3fb88321cd 100644 --- a/satpy/readers/mirs.py +++ b/satpy/readers/mirs.py @@ -95,11 +95,7 @@ def read_atms_limb_correction_coefficients(fn): coeff_str = read_atms_coeff_to_string(fn) n_chn = 22 n_fov = 96 - if n_chn < 22: - LOG.warning('Coefficient file has less than 22 channels: %s' % n_chn) - if n_fov < 96: - LOG.warning('Coefficient file has less than 96 fov: %s' % n_fov) - # make it a generator + # make the string a generator coeff_str = (line.strip() for line in coeff_str) all_coeffs = np.zeros((n_chn, n_fov, n_chn), dtype=np.float32) From 16492411e00eeac53f54bfa3e6fe6f796d16f859 Mon Sep 17 00:00:00 2001 From: Joleen Feltz Date: Tue, 18 May 2021 14:47:36 -0500 Subject: [PATCH 29/29] Simplify logic which repeatedly checked if file_type matched yaml datasets. --- satpy/readers/mirs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/satpy/readers/mirs.py b/satpy/readers/mirs.py index 3fb88321cd..eb8bee2448 100644 --- a/satpy/readers/mirs.py +++ b/satpy/readers/mirs.py @@ -435,13 +435,13 @@ def available_datasets(self, configured_datasets=None): yield is_avail, ds_info continue + yaml_info = {} if self.file_type_matches(ds_info['file_type']): handled_vars.add(ds_info['name']) + yaml_info = ds_info if ds_info['name'] == 'BT': - yaml_info = ds_info if self.file_type_matches( - ds_info['file_type']) else {} yield from self._available_btemp_datasets(yaml_info) - yield self.file_type_matches(ds_info['file_type']), ds_info + yield True, ds_info yield from self._available_new_datasets(handled_vars) def _count_channel_repeat_number(self):