Merge branch 'main' into low-moisture

.pre-commit-config.yaml

@@ -20,7 +20,7 @@ repos:
       - id: bandit
         args: [--ini, .bandit]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v0.981'  # Use the sha / tag you want to point at
+    rev: 'v0.982'  # Use the sha / tag you want to point at
     hooks:
       - id: mypy
         additional_dependencies:

doc/source/writers.rst

@@ -41,17 +41,17 @@ One common parameter across almost all Writers is ``filename`` and
       -
     * - NinJo TIFF (using ``pyninjotiff`` package)
       - :class:`ninjotiff <satpy.writers.ninjotiff.NinjoTIFFWriter>`
-      - To be deprecated (use ninjogeotiff)
+      - Deprecated from NinJo 7 (use ninjogeotiff)
       -
     * - NetCDF (Standard CF)
       - :class:`cf <satpy.writers.cf_writer.CFWriter>`
-      - Pre-alpha
+      - Beta
       - :mod:`Usage example <satpy.writers.cf_writer>`
     * - AWIPS II Tiled NetCDF4
       - :class:`awips_tiled <satpy.writers.awips_tiled.AWIPSTiledWriter>`
       - Beta
       -
-    * - GeoTIFF with NinJo tags
+    * - GeoTIFF with NinJo tags (from NinJo 7)
       - :class:`ninjogeotiff <satpy.writers.ninjogeotiff.NinJoGeoTIFFWriter>`
       - Beta
       -

satpy/composites/__init__.py

@@ -488,7 +488,7 @@ def __call__(self, projectables, nonprojectables=None, **info):
         return super(Filler, self).__call__([filled_projectable], **info)
 
 
-class MultiFiller(GenericCompositor):
+class MultiFiller(SingleBandCompositor):
     """Fix holes in projectable 1 with data from the next projectables."""
 
     def __call__(self, projectables, nonprojectables=None, **info):
@@ -501,7 +501,7 @@ def __call__(self, projectables, nonprojectables=None, **info):
             for next_projectable in info['optional_datasets']:
                 filled_projectable = filled_projectable.fillna(next_projectable)
 
-        return super(MultiFiller, self).__call__([filled_projectable], **info)
+        return super().__call__([filled_projectable], **info)
 
 
 class RGBCompositor(GenericCompositor):
@@ -924,11 +924,11 @@ class RatioSharpenedRGB(GenericCompositor):
 
     def __init__(self, *args, **kwargs):
         """Instanciate the ration sharpener."""
-        self.high_resolution_band = kwargs.pop("high_resolution_band", "red")
-        if self.high_resolution_band not in ['red', 'green', 'blue', None]:
+        self.high_resolution_color = kwargs.pop("high_resolution_band", "red")
+        if self.high_resolution_color not in ['red', 'green', 'blue', None]:
             raise ValueError("RatioSharpenedRGB.high_resolution_band must "
                              "be one of ['red', 'green', 'blue', None]. Not "
-                             "'{}'".format(self.high_resolution_band))
+                             "'{}'".format(self.high_resolution_color))
         kwargs.setdefault('common_channel_mask', False)
         super(RatioSharpenedRGB, self).__init__(*args, **kwargs)
 
@@ -945,79 +945,67 @@ def __call__(self, datasets, optional_datasets=None, **info):
             raise IncompatibleAreas('RatioSharpening requires datasets of '
                                     'the same size. Must resample first.')
 
-        new_attrs = {}
         optional_datasets = tuple() if optional_datasets is None else optional_datasets
         datasets = self.match_data_arrays(datasets + optional_datasets)
-        r = datasets[0]
-        g = datasets[1]
-        b = datasets[2]
-        if optional_datasets and self.high_resolution_band is not None:
-            LOG.debug("Sharpening image with high resolution {} band".format(self.high_resolution_band))
+        red, green, blue, new_attrs = self._get_and_sharpen_rgb_data_arrays_and_meta(datasets, optional_datasets)
+        combined_info = self._combined_sharpened_info(info, new_attrs)
+        res = super(RatioSharpenedRGB, self).__call__((red, green, blue,), **combined_info)
+        res.attrs.pop("units", None)
+        return res
+
+    def _get_and_sharpen_rgb_data_arrays_and_meta(self, datasets, optional_datasets):
+        new_attrs = {}
+        low_res_red = datasets[0]
+        low_res_green = datasets[1]
+        low_res_blue = datasets[2]
+        if optional_datasets and self.high_resolution_color is not None:
+            LOG.debug("Sharpening image with high resolution {} band".format(self.high_resolution_color))
             high_res = datasets[3]
             if 'rows_per_scan' in high_res.attrs:
                 new_attrs.setdefault('rows_per_scan', high_res.attrs['rows_per_scan'])
             new_attrs.setdefault('resolution', high_res.attrs['resolution'])
-            colors = ['red', 'green', 'blue']
-            low_res_idx = colors.index(self.high_resolution_band)
+            low_res_colors = ['red', 'green', 'blue']
+            low_resolution_index = low_res_colors.index(self.high_resolution_color)
         else:
             LOG.debug("No sharpening band specified for ratio sharpening")
             high_res = None
-            low_res_idx = 0
-
-        rgb = da.map_blocks(
-            _ratio_sharpened_rgb,
-            r.data, g.data, b.data,
-            high_res.data if high_res is not None else high_res,
-            low_resolution_index=low_res_idx,
-            new_axis=[0],
-            meta=np.array((), dtype=r.dtype),
-            dtype=r.dtype,
-            chunks=((3,),) + r.chunks,
-        )
-
-        # Collect information that is the same between the projectables
-        # we want to use the metadata from the original datasets since the
-        # new r, g, b arrays may have lost their metadata during calculations
-        combined_info = combine_metadata(*datasets)
+            low_resolution_index = 0
+
+        if high_res is not None:
+            low_res = (low_res_red, low_res_green, low_res_blue)[low_resolution_index]
+            ratio = da.map_blocks(
+                _get_sharpening_ratio,
+                high_res.data,
+                low_res.data,
+                meta=np.array((), dtype=high_res.dtype),
+                dtype=high_res.dtype,
+                chunks=high_res.chunks,
+            )
+            with xr.set_options(keep_attrs=True):
+                low_res_red = high_res if low_resolution_index == 0 else low_res_red * ratio
+                low_res_green = high_res if low_resolution_index == 1 else low_res_green * ratio
+                low_res_blue = high_res if low_resolution_index == 2 else low_res_blue * ratio
+        return low_res_red, low_res_green, low_res_blue, new_attrs
+
+    def _combined_sharpened_info(self, info, new_attrs):
+        combined_info = {}
         combined_info.update(info)
         combined_info.update(new_attrs)
         # Update that information with configured information (including name)
         combined_info.update(self.attrs)
         # Force certain pieces of metadata that we *know* to be true
         combined_info.setdefault("standard_name", "true_color")
-        combined_info["mode"] = "RGB"
-
-        rgb_data_arr = xr.DataArray(
-            rgb,
-            dims=("bands",) + r.dims,
-            coords={"bands": ["R", "G", "B"]},
-            attrs=combined_info
-        )
-
-        res = super(RatioSharpenedRGB, self).__call__((rgb_data_arr,), **combined_info)
-        res.attrs.pop("units", None)
-        return res
-
-
-def _ratio_sharpened_rgb(red, green, blue, high_res=None, low_resolution_index=0):
-    if high_res is not None:
-        low_res = (red, green, blue)[low_resolution_index]
-        ratio = high_res / low_res
-        # make ratio a no-op (multiply by 1) where the ratio is NaN or
-        # infinity or it is negative.
-        ratio[~np.isfinite(ratio) | (ratio < 0)] = 1.0
-        # we don't need ridiculously high ratios, they just make bright pixels
-        np.clip(ratio, 0, 1.5, out=ratio)
+        return combined_info
 
-        red = high_res if low_resolution_index == 0 else red * ratio
-        green = high_res if low_resolution_index == 1 else green * ratio
-        blue = high_res if low_resolution_index == 2 else blue * ratio
 
-    # combine the masks
-    mask = np.isnan(red) | np.isnan(green) | np.isnan(blue)
-    rgb = np.stack((red, green, blue))
-    rgb[:, mask] = np.nan
-    return rgb
+def _get_sharpening_ratio(high_res, low_res):
+    ratio = high_res / low_res
+    # make ratio a no-op (multiply by 1) where the ratio is NaN, infinity,
+    # or it is negative.
+    ratio[~np.isfinite(ratio) | (ratio < 0)] = 1.0
+    # we don't need ridiculously high ratios, they just make bright pixels
+    np.clip(ratio, 0, 1.5, out=ratio)
+    return ratio
 
 
 def _mean4(data, offset=(0, 0), block_id=None):
@@ -1040,7 +1028,8 @@ def _mean4(data, offset=(0, 0), block_id=None):
 
     av_data = np.pad(data, pad, 'edge')
     new_shape = (int(rows2 / 2.), 2, int(cols2 / 2.), 2)
-    data_mean = np.nanmean(av_data.reshape(new_shape), axis=(1, 3))
+    with np.errstate(invalid='ignore'):
+        data_mean = np.nanmean(av_data.reshape(new_shape), axis=(1, 3))
     data_mean = np.repeat(np.repeat(data_mean, 2, axis=0), 2, axis=1)
     data_mean = data_mean[row_offset:row_offset + rows, col_offset:col_offset + cols]
     return data_mean
@@ -1076,15 +1065,15 @@ def four_element_average_dask(d):
     def __call__(self, datasets, optional_datasets=None, **attrs):
         """Generate the composite."""
         colors = ['red', 'green', 'blue']
-        if self.high_resolution_band not in colors:
+        if self.high_resolution_color not in colors:
             raise ValueError("SelfSharpenedRGB requires at least one high resolution band, not "
-                             "'{}'".format(self.high_resolution_band))
+                             "'{}'".format(self.high_resolution_color))
 
-        high_res = datasets[colors.index(self.high_resolution_band)]
+        high_res = datasets[colors.index(self.high_resolution_color)]
         high_mean = self.four_element_average_dask(high_res)
-        red = high_mean if self.high_resolution_band == 'red' else datasets[0]
-        green = high_mean if self.high_resolution_band == 'green' else datasets[1]
-        blue = high_mean if self.high_resolution_band == 'blue' else datasets[2]
+        red = high_mean if self.high_resolution_color == 'red' else datasets[0]
+        green = high_mean if self.high_resolution_color == 'green' else datasets[1]
+        blue = high_mean if self.high_resolution_color == 'blue' else datasets[2]
         return super(SelfSharpenedRGB, self).__call__((red, green, blue), optional_datasets=(high_res,), **attrs)
 
 
@@ -1561,7 +1550,7 @@ def _get_flag_value(mask, val):
     return flag_values[index]
 
 
-class LongitudeMaskingCompositor(GenericCompositor):
+class LongitudeMaskingCompositor(SingleBandCompositor):
     """Masks areas outside defined longitudes."""
 
     def __init__(self, name, lon_min=None, lon_max=None, **kwargs):
@@ -1580,7 +1569,7 @@ def __init__(self, name, lon_min=None, lon_max=None, **kwargs):
             self.lon_min = -180.
         if not self.lon_max:
             self.lon_max = 180.
-        super(LongitudeMaskingCompositor, self).__init__(name, **kwargs)
+        super().__init__(name, **kwargs)
 
     def __call__(self, projectables, nonprojectables=None, **info):
         """Generate the composite."""
@@ -1593,4 +1582,4 @@ def __call__(self, projectables, nonprojectables=None, **info):
             lon_min_max = np.logical_or(lons >= self.lon_min, lons <= self.lon_max)
 
         masked_projectable = projectable.where(lon_min_max)
-        return super(LongitudeMaskingCompositor, self).__call__([masked_projectable], **info)
+        return super().__call__([masked_projectable], **info)

satpy/composites/ahi.py

@@ -1,6 +1,4 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright (c) 2015-2021 Satpy developers
+# Copyright (c) 2022- Satpy developers
 #
 # This file is part of satpy.
 #
@@ -15,30 +13,8 @@
 #
 # You should have received a copy of the GNU General Public License along with
 # satpy.  If not, see <http://www.gnu.org/licenses/>.
-"""Composite classes for the AHI instrument."""
+"""Composite classes for AHI."""
 
-import logging
-
-from satpy.composites import GenericCompositor
-from satpy.dataset import combine_metadata
-
-LOG = logging.getLogger(__name__)
-
-
-class GreenCorrector(GenericCompositor):
-    """Corrector of the AHI green band to compensate for the deficit of chlorophyll signal."""
-
-    def __init__(self, *args, fractions=(0.85, 0.15), **kwargs):
-        """Set default keyword argument values."""
-        # XXX: Should this be 0.93 and 0.07
-        self.fractions = fractions
-        super(GreenCorrector, self).__init__(*args, **kwargs)
-
-    def __call__(self, projectables, optional_datasets=None, **attrs):
-        """Boost vegetation effect thanks to NIR (0.8µm) band."""
-        LOG.info('Boosting vegetation on green band')
-
-        projectables = self.match_data_arrays(projectables)
-        new_green = sum(fraction * value for fraction, value in zip(self.fractions, projectables))
-        new_green.attrs = combine_metadata(*projectables)
-        return super(GreenCorrector, self).__call__((new_green,), **attrs)
+# The green corrector used to be defined here, but was moved to spectral.py
+# in Satpy 0.38 because it also applies to FCI.
+from .spectral import GreenCorrector  # noqa: F401

satpy/composites/spectral.py

@@ -0,0 +1,70 @@
+# Copyright (c) 2015-2022 Satpy developers
+#
+# This file is part of satpy.
+#
+# satpy is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# satpy is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# satpy.  If not, see <http://www.gnu.org/licenses/>.
+"""Composite classes for spectral adjustments."""
+
+import logging
+
+from satpy.composites import GenericCompositor
+from satpy.dataset import combine_metadata
+
+LOG = logging.getLogger(__name__)
+
+
+class GreenCorrector(GenericCompositor):
+    """Corrector of the FCI or AHI green band.
+
+    The green band in FCI and AHI deliberately misses the chlorophyll peak
+    in order to focus on aerosol and ash rather than on vegetation.  This
+    affects true colour RGBs, because vegetation looks brown rather than green.
+    To make vegetation look greener again, this corrector allows
+    to simulate the green band as a fraction of two or more other channels.
+
+    To be used, the composite takes two or more input channels and a parameter
+    ``fractions`` that should be a list of floats with the same length as the
+    number of channels.
+
+    For example, to simulate an FCI corrected green composite, one could use
+    a combination of 93% from the green band (vis_05) and 7% from the
+    near-infrared 0.8 µm band (vis_08)::
+
+      corrected_green:
+        compositor: !!python/name:satpy.composites.ahi.GreenCorrector
+        fractions: [0.93, 0.07]
+        prerequisites:
+          - name: vis_05
+            modifiers: [sunz_corrected, rayleigh_corrected]
+          - name: vis_08
+            modifiers: [sunz_corrected, rayleigh_corrected]
+        standard_name: toa_bidirectional_reflectance
+
+    Other examples can be found in the ``fci.yaml`` and ``ahi.yaml`` composite
+    files in the satpy distribution.
+    """
+
+    def __init__(self, *args, fractions=(0.85, 0.15), **kwargs):
+        """Set default keyword argument values."""
+        # XXX: Should this be 0.93 and 0.07
+        self.fractions = fractions
+        super(GreenCorrector, self).__init__(*args, **kwargs)
+
+    def __call__(self, projectables, optional_datasets=None, **attrs):
+        """Boost vegetation effect thanks to NIR (0.8µm) band."""
+        LOG.info('Boosting vegetation on green band')
+
+        projectables = self.match_data_arrays(projectables)
+        new_green = sum(fraction * value for fraction, value in zip(self.fractions, projectables))
+        new_green.attrs = combine_metadata(*projectables)
+        return super(GreenCorrector, self).__call__((new_green,), **attrs)

satpy/dataset/dataid.py

@@ -508,7 +508,7 @@ class DataQuery:
     """The data query object.
 
     A DataQuery can be used in Satpy to query for a Dataset. This way
-    a fully qualified DataID can be found even if some of the DataID
+    a fully qualified DataID can be found even if some DataID
     elements are unknown. In this case a `*` signifies something that is
     unknown or not applicable to the requested Dataset.
     """

satpy/etc/composites/ahi.yaml

@@ -16,7 +16,7 @@ modifiers:
 
 composites:
   green:
-    compositor: !!python/name:satpy.composites.ahi.GreenCorrector
+    compositor: !!python/name:satpy.composites.spectral.GreenCorrector
     # FUTURE: Set a wavelength...see what happens. Dependency finding
     #         probably wouldn't work.
     prerequisites:
@@ -31,7 +31,7 @@ composites:
   green_true_color_reproduction:
     # JMA True Color Reproduction green band
     # http://www.jma.go.jp/jma/jma-eng/satellite/introduction/TCR.html
-    compositor: !!python/name:satpy.composites.ahi.GreenCorrector
+    compositor: !!python/name:satpy.composites.spectral.GreenCorrector
     fractions: [0.6321, 0.2928, 0.0751]
     prerequisites:
     - name: B02
@@ -43,7 +43,7 @@ composites:
     standard_name: none
 
   green_nocorr:
-    compositor: !!python/name:satpy.composites.ahi.GreenCorrector
+    compositor: !!python/name:satpy.composites.spectral.GreenCorrector
     # FUTURE: Set a wavelength...see what happens. Dependency finding
     #         probably wouldn't work.
     prerequisites: