Merge branch 'main' into feature_median_filter

AUTHORS.md

@@ -35,6 +35,7 @@ The following people have made contributions to this project:
 - [Gerrit Holl (gerritholl)](https://github.com/gerritholl) - Deutscher Wetterdienst
 - [David Hoese (djhoese)](https://github.com/djhoese)
 - [Marc Honnorat (honnorat)](https://github.com/honnorat)
+- [Lloyd Hughes (system123)](https://github.com/system123)
 - [Mikhail Itkin (mitkin)](https://github.com/mitkin)
 - [Tommy Jasmin (tommyjasmin)](https://github.com/tommyjasmin)
 - [Jactry Zeng](https://github.com/jactry)

continuous_integration/environment.yaml

@@ -10,6 +10,7 @@ dependencies:
   - appdirs
   - toolz
   - Cython
+  - numba
   - sphinx
   - cartopy
   - panel>=0.12.7

doc/rtd_environment.yml

@@ -12,6 +12,7 @@ dependencies:
   # 2.19.1 seems to cause library linking issues
   - eccodes>=2.20
   - graphviz
+  - numba
   - numpy
   - pillow
   - pooch

doc/source/composites.rst

@@ -173,7 +173,7 @@ In the case below, the image shows its day portion and day/night
 transition with night portion blacked-out instead of transparent::
 
     >>> from satpy.composites import DayNightCompositor
-    >>> compositor = DayNightCompositor("dnc", lim_low=85., lim_high=88., day_night="day_only", need_alpha=False)
+    >>> compositor = DayNightCompositor("dnc", lim_low=85., lim_high=88., day_night="day_only", include_alpha=False)
     >>> composite = compositor([local_scene['true_color'])
 
 RealisticColors

doc/source/config.rst

@@ -258,6 +258,23 @@ as part of the :func:`~satpy.modifiers.angles.get_angles` and
 used by multiple modifiers and composites including the default rayleigh
 correction.
 
+Clipping Negative Infrared Radiances
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+* **Environment variable**: ``SATPY_READERS__CLIP_NEGATIVE_RADIANCES``
+* **YAML/Config Key**: ``readers.clip_negative_radiances``
+* **Default**: False
+
+Whether to clip negative infrared radiances to the minimum allowable value before
+computing the brightness temperature.
+If ``clip_negative_radiances=False``, pixels with negative radiances will have
+``np.nan`` brightness temperatures.
+
+Clipping of negative radiances is currently implemented for the following readers:
+
+* ``abi_l1b``
+
+
 Temporary Directory
 ^^^^^^^^^^^^^^^^^^^
 

satpy/_config.py

@@ -62,6 +62,9 @@ def impr_files(module_name: str) -> Path:
     'demo_data_dir': '.',
     'download_aux': True,
     'sensor_angles_position_preference': 'actual',
+    'readers': {
+        'clip_negative_radiances': False,
+    },
 }
 
 # Satpy main configuration object

satpy/_scene_converters.py

@@ -0,0 +1,124 @@
+# Copyright (c) 2023 Satpy developers
+#
+# This file is part of satpy.
+#
+# satpy is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# satpy is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# satpy.  If not, see <http://www.gnu.org/licenses/>.
+"""Helper functions for converting the Scene object to some other object."""
+
+import xarray as xr
+
+from satpy.dataset import DataID
+
+
+def _get_dataarrays_from_identifiers(scn, identifiers):
+    """Return a list of DataArray based on a single or list of identifiers.
+
+    An identifier can be a DataID or a string with name of a valid DataID.
+    """
+    if isinstance(identifiers, (str, DataID)):
+        identifiers = [identifiers]
+
+    if identifiers is not None:
+        dataarrays = [scn[ds] for ds in identifiers]
+    else:
+        dataarrays = [scn._datasets.get(ds) for ds in scn._wishlist]
+        dataarrays = [dataarray for dataarray in dataarrays if dataarray is not None]
+    return dataarrays
+
+
+def to_xarray(scn,
+              datasets=None,  # DataID
+              header_attrs=None,
+              exclude_attrs=None,
+              flatten_attrs=False,
+              pretty=True,
+              include_lonlats=True,
+              epoch=None,
+              include_orig_name=True,
+              numeric_name_prefix='CHANNEL_'):
+    """Merge all xr.DataArray(s) of a satpy.Scene to a CF-compliant xarray object.
+
+    If all Scene DataArrays are on the same area, it returns an xr.Dataset.
+    If Scene DataArrays are on different areas, currently it fails, although
+    in future we might return a DataTree object, grouped by area.
+
+    Parameters
+    ----------
+    scn: satpy.Scene
+        Satpy Scene.
+    datasets (iterable):
+        List of Satpy Scene datasets to include in the output xr.Dataset.
+        Elements can be string name, a wavelength as a number, a DataID,
+        or DataQuery object.
+        If None (the default), it include all loaded Scene datasets.
+    header_attrs:
+        Global attributes of the output xr.Dataset.
+    epoch (str):
+        Reference time for encoding the time coordinates (if available).
+        Example format: "seconds since 1970-01-01 00:00:00".
+        If None, the default reference time is retrieved using "from satpy.cf_writer import EPOCH"
+    flatten_attrs (bool):
+        If True, flatten dict-type attributes.
+    exclude_attrs (list):
+        List of xr.DataArray attribute names to be excluded.
+    include_lonlats (bool):
+        If True, it includes 'latitude' and 'longitude' coordinates.
+        If the 'area' attribute is a SwathDefinition, it always includes
+        latitude and longitude coordinates.
+    pretty (bool):
+        Don't modify coordinate names, if possible. Makes the file prettier,
+        but possibly less consistent.
+    include_orig_name (bool).
+        Include the original dataset name as a variable attribute in the xr.Dataset.
+    numeric_name_prefix (str):
+        Prefix to add the each variable with name starting with a digit.
+        Use '' or None to leave this out.
+
+    Returns
+    -------
+    ds, xr.Dataset
+        A CF-compliant xr.Dataset
+
+    """
+    from satpy.writers.cf_writer import EPOCH, collect_cf_datasets
+
+    if epoch is None:
+        epoch = EPOCH
+
+    # Get list of DataArrays
+    if datasets is None:
+        datasets = list(scn.keys())  # list all loaded DataIDs
+    list_dataarrays = _get_dataarrays_from_identifiers(scn, datasets)
+
+    # Check that some DataArray could be returned
+    if len(list_dataarrays) == 0:
+        return xr.Dataset()
+
+    # Collect xr.Dataset for each group
+    grouped_datasets, header_attrs = collect_cf_datasets(list_dataarrays=list_dataarrays,
+                                                         header_attrs=header_attrs,
+                                                         exclude_attrs=exclude_attrs,
+                                                         flatten_attrs=flatten_attrs,
+                                                         pretty=pretty,
+                                                         include_lonlats=include_lonlats,
+                                                         epoch=epoch,
+                                                         include_orig_name=include_orig_name,
+                                                         numeric_name_prefix=numeric_name_prefix,
+                                                         groups=None)
+    if len(grouped_datasets) == 1:
+        ds = grouped_datasets[None]
+        return ds
+    else:
+        msg = """The Scene object contains datasets with different areas.
+                  Resample the Scene to have matching dimensions using i.e. scn.resample(resampler="native") """
+        raise NotImplementedError(msg)

satpy/composites/__init__.py

@@ -1039,15 +1039,36 @@ class RatioSharpenedRGB(GenericCompositor):
         new_G = G * ratio
         new_B = B * ratio
 
+    In some cases, there could be multiple high resolution bands::
+
+        R_lo -  1000m resolution - shape=(2000, 2000)
+        G_hi - 500m resolution - shape=(4000, 4000)
+        B - 1000m resolution - shape=(2000, 2000)
+        R_hi -  500m resolution - shape=(4000, 4000)
+
+    To avoid the green band getting involved in calculating ratio or sharpening,
+    add "neutral_resolution_band: green" in the YAML config file. This way
+    only the blue band will get sharpened::
+
+        ratio = R_hi / R_lo
+        new_R = R_hi
+        new_G = G_hi
+        new_B = B * ratio
+
     """
 
     def __init__(self, *args, **kwargs):
         """Instanciate the ration sharpener."""
         self.high_resolution_color = kwargs.pop("high_resolution_band", "red")
+        self.neutral_resolution_color = kwargs.pop("neutral_resolution_band", None)
         if self.high_resolution_color not in ['red', 'green', 'blue', None]:
             raise ValueError("RatioSharpenedRGB.high_resolution_band must "
                              "be one of ['red', 'green', 'blue', None]. Not "
                              "'{}'".format(self.high_resolution_color))
+        if self.neutral_resolution_color not in ['red', 'green', 'blue', None]:
+            raise ValueError("RatioSharpenedRGB.neutral_resolution_band must "
+                             "be one of ['red', 'green', 'blue', None]. Not "
+                             "'{}'".format(self.neutral_resolution_color))
         super(RatioSharpenedRGB, self).__init__(*args, **kwargs)
 
     def __call__(self, datasets, optional_datasets=None, **info):
@@ -1082,28 +1103,33 @@ def _get_and_sharpen_rgb_data_arrays_and_meta(self, datasets, optional_datasets)
             if 'rows_per_scan' in high_res.attrs:
                 new_attrs.setdefault('rows_per_scan', high_res.attrs['rows_per_scan'])
             new_attrs.setdefault('resolution', high_res.attrs['resolution'])
-            low_res_colors = ['red', 'green', 'blue']
-            low_resolution_index = low_res_colors.index(self.high_resolution_color)
+
         else:
             LOG.debug("No sharpening band specified for ratio sharpening")
             high_res = None
-            low_resolution_index = 0
 
+        bands = {'red': low_res_red, 'green': low_res_green, 'blue': low_res_blue}
         if high_res is not None:
-            low_res = (low_res_red, low_res_green, low_res_blue)[low_resolution_index]
-            ratio = da.map_blocks(
-                _get_sharpening_ratio,
-                high_res.data,
-                low_res.data,
-                meta=np.array((), dtype=high_res.dtype),
-                dtype=high_res.dtype,
-                chunks=high_res.chunks,
-            )
-            with xr.set_options(keep_attrs=True):
-                low_res_red = high_res if low_resolution_index == 0 else low_res_red * ratio
-                low_res_green = high_res if low_resolution_index == 1 else low_res_green * ratio
-                low_res_blue = high_res if low_resolution_index == 2 else low_res_blue * ratio
-        return low_res_red, low_res_green, low_res_blue, new_attrs
+            self._sharpen_bands_with_high_res(bands, high_res)
+
+        return bands['red'], bands['green'], bands['blue'], new_attrs
+
+    def _sharpen_bands_with_high_res(self, bands, high_res):
+        ratio = da.map_blocks(
+            _get_sharpening_ratio,
+            high_res.data,
+            bands[self.high_resolution_color].data,
+            meta=np.array((), dtype=high_res.dtype),
+            dtype=high_res.dtype,
+            chunks=high_res.chunks,
+        )
+
+        bands[self.high_resolution_color] = high_res
+
+        with xr.set_options(keep_attrs=True):
+            for color in bands.keys():
+                if color != self.neutral_resolution_color and color != self.high_resolution_color:
+                    bands[color] = bands[color] * ratio
 
     def _combined_sharpened_info(self, info, new_attrs):
         combined_info = {}

satpy/etc/readers/gms5-vissr_l1b.yaml

@@ -0,0 +1,99 @@
+reader:
+  name: gms5-vissr_l1b
+  short_name: GMS-5 VISSR L1b
+  long_name: GMS-5 VISSR Level 1b
+  description: >
+    Reader for GMS-5 VISSR Level 1b data. References:
+
+    - https://www.data.jma.go.jp/mscweb/en/operation/fig/VISSR_FORMAT_GMS-5.pdf
+    - https://www.data.jma.go.jp/mscweb/en/operation/fig/GMS_Users_Guide_3rd_Edition_Rev1.pdf
+
+  status: Alpha
+  supports_fsspec: true
+  sensors: [gms5-vissr]
+  default_channels: []
+  reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader
+
+file_types:
+    gms5_vissr_vis:
+        file_reader: !!python/name:satpy.readers.gms.gms5_vissr_l1b.GMS5VISSRFileHandler
+        file_patterns:
+        - 'VISSR_{start_time:%Y%m%d_%H%M}_VIS.{mode}.IMG'
+        - 'VISSR_{start_time:%Y%m%d_%H%M}_VIS.{mode}.IMG.gz'
+
+    gms5_vissr_ir1:
+        file_reader: !!python/name:satpy.readers.gms.gms5_vissr_l1b.GMS5VISSRFileHandler
+        file_patterns:
+        - 'VISSR_{start_time:%Y%m%d_%H%M}_IR1.{mode}.IMG'
+        - 'VISSR_{start_time:%Y%m%d_%H%M}_IR1.{mode}.IMG.gz'
+
+    gms5_vissr_ir2:
+        file_reader: !!python/name:satpy.readers.gms.gms5_vissr_l1b.GMS5VISSRFileHandler
+        file_patterns:
+        - 'VISSR_{start_time:%Y%m%d_%H%M}_IR2.{mode}.IMG'
+        - 'VISSR_{start_time:%Y%m%d_%H%M}_IR2.{mode}.IMG.gz'
+
+
+    gms5_vissr_ir3:
+        file_reader: !!python/name:satpy.readers.gms.gms5_vissr_l1b.GMS5VISSRFileHandler
+        file_patterns:
+        - 'VISSR_{start_time:%Y%m%d_%H%M}_IR3.{mode}.IMG'
+        - 'VISSR_{start_time:%Y%m%d_%H%M}_IR3.{mode}.IMG.gz'
+
+
+datasets:
+  VIS:
+    name: VIS
+    sensor: gms5-vissr
+    wavelength: [0.55, 0.73, 0.9]
+    resolution: 1250
+    calibration:
+      counts:
+        standard_name: counts
+        units: 1
+      reflectance:
+        standard_name: toa_bidirectional_reflectance
+        units: "%"
+    file_type: gms5_vissr_vis
+
+  IR1:
+    name: IR1
+    sensor: gms5-vissr
+    wavelength: [10.5, 11.0, 11.5]
+    resolution: 5000
+    calibration:
+      counts:
+        standard_name: counts
+        units: 1
+      brightness_temperature:
+        standard_name: toa_brightness_temperature
+        units: K
+    file_type: gms5_vissr_ir1
+
+  IR2:
+    name: IR2
+    sensor: gms5-vissr
+    wavelength: [11.5, 12.0, 12.5]
+    resolution: 5000
+    calibration:
+      counts:
+        standard_name: counts
+        units: 1
+      brightness_temperature:
+        standard_name: toa_brightness_temperature
+        units: K
+    file_type: gms5_vissr_ir2
+
+  IR3:
+    name: IR3
+    sensor: gms5-vissr
+    wavelength: [6.5, 6.75, 7.0]
+    resolution: 5000
+    calibration:
+      counts:
+        standard_name: counts
+        units: 1
+      brightness_temperature:
+        standard_name: toa_brightness_temperature
+        units: K
+    file_type: gms5_vissr_ir3