From 983961bfb06cdb80ae9dcebd7ad86fe6cca13382 Mon Sep 17 00:00:00 2001 From: Tommy Jasmin Date: Tue, 3 Aug 2021 09:58:53 -0500 Subject: [PATCH] Add raw 'counts' calibration to 'abi_l1b' reader (#1692) * Raw count calibration needed for python ADDE support * Tune RAW calibration and add a test [#1692] * Refactor a bit to please Codebeat * Forgot to lint it, minor edits * Docstring formatting fixes * Fix bug introduced by raw units matching albedo units * Forgot to update unit test for raw calibration * Incorporate suggested changes from PR 1692 * Clarity code in reader, add explanation for assertion test * Lose an unnecessary else * Switch assertTrue to clearer assertEqual in satpy/tests/reader_tests/test_abi_l1b.py Co-authored-by: David Hoese --- satpy/etc/readers/abi_l1b.yaml | 48 +++++++++++++++++++++ satpy/readers/abi_l1b.py | 54 ++++++++++++++++++------ satpy/tests/reader_tests/test_abi_l1b.py | 37 ++++++++++++++++ 3 files changed, 126 insertions(+), 13 deletions(-) diff --git a/satpy/etc/readers/abi_l1b.yaml b/satpy/etc/readers/abi_l1b.yaml index 5bc4913e50..eb1df4ebce 100644 --- a/satpy/etc/readers/abi_l1b.yaml +++ b/satpy/etc/readers/abi_l1b.yaml @@ -102,6 +102,9 @@ datasets: reflectance: standard_name: toa_bidirectional_reflectance units: "%" + counts: + standard_name: counts + units: "1" file_type: c01 C02: @@ -115,6 +118,9 @@ datasets: reflectance: standard_name: toa_bidirectional_reflectance units: "%" + counts: + standard_name: counts + units: "1" file_type: c02 C03: @@ -128,6 +134,9 @@ datasets: reflectance: standard_name: toa_bidirectional_reflectance units: "%" + counts: + standard_name: counts + units: "1" file_type: c03 C04: @@ -141,6 +150,9 @@ datasets: reflectance: standard_name: toa_bidirectional_reflectance units: "%" + counts: + standard_name: counts + units: "1" file_type: c04 C05: @@ -154,6 +166,9 @@ datasets: reflectance: standard_name: toa_bidirectional_reflectance units: "%" + counts: + standard_name: counts + units: "1" file_type: c05 C06: @@ -167,6 +182,9 @@ datasets: reflectance: standard_name: toa_bidirectional_reflectance units: "%" + counts: + standard_name: counts + units: "1" file_type: c06 C07: @@ -180,6 +198,9 @@ datasets: brightness_temperature: standard_name: toa_brightness_temperature units: K + counts: + standard_name: counts + units: "1" file_type: c07 C08: @@ -193,6 +214,9 @@ datasets: brightness_temperature: standard_name: toa_brightness_temperature units: K + counts: + standard_name: counts + units: "1" file_type: c08 C09: @@ -206,6 +230,9 @@ datasets: brightness_temperature: standard_name: toa_brightness_temperature units: K + counts: + standard_name: counts + units: "1" file_type: c09 C10: @@ -219,6 +246,9 @@ datasets: brightness_temperature: standard_name: toa_brightness_temperature units: K + counts: + standard_name: counts + units: "1" file_type: c10 C11: @@ -232,6 +262,9 @@ datasets: brightness_temperature: standard_name: toa_brightness_temperature units: K + counts: + standard_name: counts + units: "1" file_type: c11 C12: @@ -245,6 +278,9 @@ datasets: brightness_temperature: standard_name: toa_brightness_temperature units: K + counts: + standard_name: counts + units: "1" file_type: c12 C13: @@ -258,6 +294,9 @@ datasets: brightness_temperature: standard_name: toa_brightness_temperature units: K + counts: + standard_name: counts + units: "1" file_type: c13 C14: @@ -271,6 +310,9 @@ datasets: brightness_temperature: standard_name: toa_brightness_temperature units: K + counts: + standard_name: counts + units: "1" file_type: c14 C15: @@ -284,6 +326,9 @@ datasets: brightness_temperature: standard_name: toa_brightness_temperature units: K + counts: + standard_name: counts + units: "1" file_type: c15 C16: @@ -297,4 +342,7 @@ datasets: brightness_temperature: standard_name: toa_brightness_temperature units: K + counts: + standard_name: counts + units: "1" file_type: c16 diff --git a/satpy/readers/abi_l1b.py b/satpy/readers/abi_l1b.py index 5102d99a30..3c55115905 100644 --- a/satpy/readers/abi_l1b.py +++ b/satpy/readers/abi_l1b.py @@ -38,21 +38,26 @@ class NC_ABI_L1B(NC_ABI_BASE): def get_dataset(self, key, info): """Load a dataset.""" logger.debug('Reading in get_dataset %s.', key['name']) + # For raw cal, don't apply scale and offset, return raw file counts + if key['calibration'] == 'counts': + return self._raw_calibrate(self.nc['Rad']) radiances = self['Rad'] - if key['calibration'] == 'reflectance': - logger.debug("Calibrating to reflectances") - res = self._vis_calibrate(radiances) - elif key['calibration'] == 'brightness_temperature': - logger.debug("Calibrating to brightness temperatures") - res = self._ir_calibrate(radiances) - elif key['calibration'] != 'radiance': + # mapping of calibration types to calibration functions + cal_dictionary = { + 'reflectance': self._vis_calibrate, + 'brightness_temperature': self._ir_calibrate, + 'radiance': self._rad_calibrate + } + + try: + func = cal_dictionary[key['calibration']] + res = func(radiances) + except KeyError: raise ValueError("Unknown calibration '{}'".format(key['calibration'])) - else: - res = radiances # convert to satpy standard units - if res.attrs['units'] == '1': + if res.attrs['units'] == '1' and key['calibration'] != 'counts': res *= 100 res.attrs['units'] = '%' @@ -73,9 +78,11 @@ def get_dataset(self, key, info): res.attrs.update(key.to_dict()) # remove attributes that could be confusing later - res.attrs.pop('_FillValue', None) - res.attrs.pop('scale_factor', None) - res.attrs.pop('add_offset', None) + # if calibration type is raw counts, we leave them in + if key['calibration'] != 'counts': + res.attrs.pop('_FillValue', None) + res.attrs.pop('scale_factor', None) + res.attrs.pop('add_offset', None) res.attrs.pop('_Unsigned', None) res.attrs.pop('ancillary_variables', None) # Can't currently load DQF # although we could compute these, we'd have to update in calibration @@ -91,7 +98,28 @@ def get_dataset(self, key, info): for attr in ('fusion_args',): if attr in self.nc.attrs: res.attrs[attr] = self.nc.attrs[attr] + return res + + def _rad_calibrate(self, data): + """Calibrate any channel to radiances. + + This no-op method is just to keep the flow consistent - + each valid cal type results in a calibration method call + """ + res = data + res.attrs = data.attrs + return res + + def _raw_calibrate(self, data): + """Calibrate any channel to raw counts. + Useful for cases where a copy requires no calibration. + """ + res = data + res.attrs = data.attrs + res.attrs['units'] = '1' + res.attrs['long_name'] = 'Raw Counts' + res.attrs['standard_name'] = 'counts' return res def _vis_calibrate(self, data): diff --git a/satpy/tests/reader_tests/test_abi_l1b.py b/satpy/tests/reader_tests/test_abi_l1b.py index 6ea4a3599e..725121796e 100644 --- a/satpy/tests/reader_tests/test_abi_l1b.py +++ b/satpy/tests/reader_tests/test_abi_l1b.py @@ -269,6 +269,43 @@ def test_vis_calibrate(self): 'Bidirectional Reflectance') +class Test_NC_ABI_L1B_raw_cal(Test_NC_ABI_L1B_Base): + """Test the NC_ABI_L1B reader raw calibration.""" + + def setUp(self): + """Create fake data for the tests.""" + rad_data = (np.arange(10.).reshape((2, 5)) + 1.) + rad_data = (rad_data + 1.) / 0.5 + rad_data = rad_data.astype(np.int16) + rad = xr.DataArray( + rad_data, + dims=('y', 'x'), + attrs={ + 'scale_factor': 0.5, + 'add_offset': -1., + '_FillValue': 20, + } + ) + super(Test_NC_ABI_L1B_raw_cal, self).setUp(rad=rad) + + def test_raw_calibrate(self): + """Test RAW calibration.""" + res = self.reader.get_dataset( + make_dataid(name='C05', calibration='counts'), {}) + + # We expect the raw data to be unchanged + expected = res.data + self.assertTrue(np.allclose(res.data, expected, equal_nan=True)) + self.assertEqual(res.data.dtype, np.int16, "int16 data type expected") + self.assertIn('scale_factor', res.attrs) + self.assertIn('add_offset', res.attrs) + self.assertIn('_FillValue', res.attrs) + self.assertEqual(res.attrs['standard_name'], + 'counts') + self.assertEqual(res.attrs['long_name'], + 'Raw Counts') + + class Test_NC_ABI_File(unittest.TestCase): """Test file opening."""