Add raw 'counts' calibration to 'abi_l1b' reader (#1692)

* Raw count calibration needed for python ADDE support

* Tune RAW calibration and add a test [#1692]

* Refactor a bit to please Codebeat

* Forgot to lint it, minor edits

* Docstring formatting fixes

* Fix bug introduced by raw units matching albedo units

* Forgot to update unit test for raw calibration

* Incorporate suggested changes from PR 1692

* Clarity code in reader, add explanation for assertion test

* Lose an unnecessary else

* Switch assertTrue to clearer assertEqual in satpy/tests/reader_tests/test_abi_l1b.py

Co-authored-by: David Hoese <david.hoese@ssec.wisc.edu>

satpy/etc/readers/abi_l1b.yaml

@@ -102,6 +102,9 @@ datasets:
       reflectance:
         standard_name: toa_bidirectional_reflectance
         units: "%"
+      counts:
+        standard_name: counts
+        units: "1"
     file_type: c01
 
   C02:
@@ -115,6 +118,9 @@ datasets:
       reflectance:
         standard_name: toa_bidirectional_reflectance
         units: "%"
+      counts:
+        standard_name: counts
+        units: "1"
     file_type: c02
 
   C03:
@@ -128,6 +134,9 @@ datasets:
       reflectance:
         standard_name: toa_bidirectional_reflectance
         units: "%"
+      counts:
+        standard_name: counts
+        units: "1"
     file_type: c03
 
   C04:
@@ -141,6 +150,9 @@ datasets:
       reflectance:
         standard_name: toa_bidirectional_reflectance
         units: "%"
+      counts:
+        standard_name: counts
+        units: "1"
     file_type: c04
 
   C05:
@@ -154,6 +166,9 @@ datasets:
       reflectance:
         standard_name: toa_bidirectional_reflectance
         units: "%"
+      counts:
+        standard_name: counts
+        units: "1"
     file_type: c05
 
   C06:
@@ -167,6 +182,9 @@ datasets:
       reflectance:
         standard_name: toa_bidirectional_reflectance
         units: "%"
+      counts:
+        standard_name: counts
+        units: "1"
     file_type: c06
 
   C07:
@@ -180,6 +198,9 @@ datasets:
       brightness_temperature:
         standard_name: toa_brightness_temperature
         units: K
+      counts:
+        standard_name: counts
+        units: "1"
     file_type: c07
 
   C08:
@@ -193,6 +214,9 @@ datasets:
       brightness_temperature:
         standard_name: toa_brightness_temperature
         units: K
+      counts:
+        standard_name: counts
+        units: "1"
     file_type: c08
 
   C09:
@@ -206,6 +230,9 @@ datasets:
       brightness_temperature:
         standard_name: toa_brightness_temperature
         units: K
+      counts:
+        standard_name: counts
+        units: "1"
     file_type: c09
 
   C10:
@@ -219,6 +246,9 @@ datasets:
       brightness_temperature:
         standard_name: toa_brightness_temperature
         units: K
+      counts:
+        standard_name: counts
+        units: "1"
     file_type: c10
 
   C11:
@@ -232,6 +262,9 @@ datasets:
       brightness_temperature:
         standard_name: toa_brightness_temperature
         units: K
+      counts:
+        standard_name: counts
+        units: "1"
     file_type: c11
 
   C12:
@@ -245,6 +278,9 @@ datasets:
       brightness_temperature:
         standard_name: toa_brightness_temperature
         units: K
+      counts:
+        standard_name: counts
+        units: "1"
     file_type: c12
 
   C13:
@@ -258,6 +294,9 @@ datasets:
       brightness_temperature:
         standard_name: toa_brightness_temperature
         units: K
+      counts:
+        standard_name: counts
+        units: "1"
     file_type: c13
 
   C14:
@@ -271,6 +310,9 @@ datasets:
       brightness_temperature:
         standard_name: toa_brightness_temperature
         units: K
+      counts:
+        standard_name: counts
+        units: "1"
     file_type: c14
 
   C15:
@@ -284,6 +326,9 @@ datasets:
       brightness_temperature:
         standard_name: toa_brightness_temperature
         units: K
+      counts:
+        standard_name: counts
+        units: "1"
     file_type: c15
 
   C16:
@@ -297,4 +342,7 @@ datasets:
       brightness_temperature:
         standard_name: toa_brightness_temperature
         units: K
+      counts:
+        standard_name: counts
+        units: "1"
     file_type: c16

satpy/readers/abi_l1b.py

@@ -38,21 +38,26 @@ class NC_ABI_L1B(NC_ABI_BASE):
     def get_dataset(self, key, info):
         """Load a dataset."""
         logger.debug('Reading in get_dataset %s.', key['name'])
+        # For raw cal, don't apply scale and offset, return raw file counts
+        if key['calibration'] == 'counts':
+            return self._raw_calibrate(self.nc['Rad'])
         radiances = self['Rad']
 
-        if key['calibration'] == 'reflectance':
-            logger.debug("Calibrating to reflectances")
-            res = self._vis_calibrate(radiances)
-        elif key['calibration'] == 'brightness_temperature':
-            logger.debug("Calibrating to brightness temperatures")
-            res = self._ir_calibrate(radiances)
-        elif key['calibration'] != 'radiance':
+        # mapping of calibration types to calibration functions
+        cal_dictionary = {
+            'reflectance': self._vis_calibrate,
+            'brightness_temperature': self._ir_calibrate,
+            'radiance': self._rad_calibrate
+        }
+
+        try:
+            func = cal_dictionary[key['calibration']]
+            res = func(radiances)
+        except KeyError:
             raise ValueError("Unknown calibration '{}'".format(key['calibration']))
-        else:
-            res = radiances
 
         # convert to satpy standard units
-        if res.attrs['units'] == '1':
+        if res.attrs['units'] == '1' and key['calibration'] != 'counts':
             res *= 100
             res.attrs['units'] = '%'
 
@@ -73,9 +78,11 @@ def get_dataset(self, key, info):
 
         res.attrs.update(key.to_dict())
         # remove attributes that could be confusing later
-        res.attrs.pop('_FillValue', None)
-        res.attrs.pop('scale_factor', None)
-        res.attrs.pop('add_offset', None)
+        # if calibration type is raw counts, we leave them in
+        if key['calibration'] != 'counts':
+            res.attrs.pop('_FillValue', None)
+            res.attrs.pop('scale_factor', None)
+            res.attrs.pop('add_offset', None)
         res.attrs.pop('_Unsigned', None)
         res.attrs.pop('ancillary_variables', None)  # Can't currently load DQF
         # although we could compute these, we'd have to update in calibration
@@ -91,7 +98,28 @@ def get_dataset(self, key, info):
         for attr in ('fusion_args',):
             if attr in self.nc.attrs:
                 res.attrs[attr] = self.nc.attrs[attr]
+        return res
+
+    def _rad_calibrate(self, data):
+        """Calibrate any channel to radiances.
+
+        This no-op method is just to keep the flow consistent -
+        each valid cal type results in a calibration method call
+        """
+        res = data
+        res.attrs = data.attrs
+        return res
+
+    def _raw_calibrate(self, data):
+        """Calibrate any channel to raw counts.
 
+        Useful for cases where a copy requires no calibration.
+        """
+        res = data
+        res.attrs = data.attrs
+        res.attrs['units'] = '1'
+        res.attrs['long_name'] = 'Raw Counts'
+        res.attrs['standard_name'] = 'counts'
         return res
 
     def _vis_calibrate(self, data):

satpy/tests/reader_tests/test_abi_l1b.py

@@ -269,6 +269,43 @@ def test_vis_calibrate(self):
                          'Bidirectional Reflectance')
 
 
+class Test_NC_ABI_L1B_raw_cal(Test_NC_ABI_L1B_Base):
+    """Test the NC_ABI_L1B reader raw calibration."""
+
+    def setUp(self):
+        """Create fake data for the tests."""
+        rad_data = (np.arange(10.).reshape((2, 5)) + 1.)
+        rad_data = (rad_data + 1.) / 0.5
+        rad_data = rad_data.astype(np.int16)
+        rad = xr.DataArray(
+            rad_data,
+            dims=('y', 'x'),
+            attrs={
+                'scale_factor': 0.5,
+                'add_offset': -1.,
+                '_FillValue': 20,
+            }
+        )
+        super(Test_NC_ABI_L1B_raw_cal, self).setUp(rad=rad)
+
+    def test_raw_calibrate(self):
+        """Test RAW calibration."""
+        res = self.reader.get_dataset(
+            make_dataid(name='C05', calibration='counts'), {})
+
+        # We expect the raw data to be unchanged
+        expected = res.data
+        self.assertTrue(np.allclose(res.data, expected, equal_nan=True))
+        self.assertEqual(res.data.dtype, np.int16, "int16 data type expected")
+        self.assertIn('scale_factor', res.attrs)
+        self.assertIn('add_offset', res.attrs)
+        self.assertIn('_FillValue', res.attrs)
+        self.assertEqual(res.attrs['standard_name'],
+                         'counts')
+        self.assertEqual(res.attrs['long_name'],
+                         'Raw Counts')
+
+
 class Test_NC_ABI_File(unittest.TestCase):
     """Test file opening."""