Merge branch 'main' into fix-cf-numpy2

.github/workflows/ci.yaml

@@ -82,9 +82,6 @@ jobs:
       - name: Update environment
         run: mamba env update -n test-environment -f continuous_integration/environment.yaml
         if: steps.cache.outputs.cache-hit != 'true'
-      - name: Update environment - libnetcdf
-        run: mamba install -y -n test-environment libnetcdf=4.9.2=nompi_h5902ca5_107
-        if: runner.os == 'Windows'
 
       - name: Install unstable dependencies
         if: matrix.experimental == true

.github/workflows/deploy-sdist.yaml

@@ -1,6 +1,8 @@
 name: Deploy sdist
 
 on:
+  push:
+  pull_request:
   release:
     types:
       - published
@@ -15,7 +17,9 @@ jobs:
 
       - name: Create sdist
         shell: bash -l {0}
-        run: python setup.py sdist
+        run: |
+          python -m pip install -q build
+          python -m build -s
 
       - name: Publish package to PyPI
         if: github.event.action == 'published'

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
         additional_dependencies: [flake8-docstrings, flake8-debugger, flake8-bugbear, mccabe]
         args: [--max-complexity, "10"]
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -28,7 +28,7 @@ repos:
           - types-pkg-resources
           - types-PyYAML
           - types-requests
-        args: ["--python-version", "3.8", "--ignore-missing-imports"]
+        args: ["--python-version", "3.9", "--ignore-missing-imports"]
   - repo: https://github.com/pycqa/isort
     rev: 5.12.0
     hooks:

MANIFEST.in

@@ -1,6 +1,16 @@
-include doc/Makefile
-include doc/source/*
-include doc/examples/*.py
+prune *
+exclude *
+graft doc
+recursive-exclude doc/build *
+graft satpy
 include LICENSE.txt
 include README.rst
+include AUTHORS.md
+include CHANGELOG.md
+include SECURITY.md
+include CITATION
 include satpy/version.py
+include pyproject.toml
+include setup.py
+include setup.cfg
+global-exclude *.py[cod]

pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.2", 'setuptools_scm_git_archive']
+requires = ["setuptools>=60", "wheel", "setuptools_scm[toml]>=8.0"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools_scm]

satpy/_compat.py

@@ -17,80 +17,11 @@
 # satpy.  If not, see <http://www.gnu.org/licenses/>.
 """Backports and compatibility fixes for satpy."""
 
-from threading import RLock
-
-_NOT_FOUND = object()
-
-
-class CachedPropertyBackport:
-    """Backport of cached_property from Python-3.8.
-
-    Source: https://github.com/python/cpython/blob/v3.8.0/Lib/functools.py#L930
-    """
-
-    def __init__(self, func):  # noqa
-        self.func = func
-        self.attrname = None
-        self.__doc__ = func.__doc__
-        self.lock = RLock()
-
-    def __set_name__(self, owner, name):  # noqa
-        if self.attrname is None:
-            self.attrname = name
-        elif name != self.attrname:
-            raise TypeError(
-                "Cannot assign the same cached_property to two different names "
-                f"({self.attrname!r} and {name!r})."
-            )
-
-    def __get__(self, instance, owner=None):  # noqa
-        if instance is None:
-            return self
-        if self.attrname is None:
-            raise TypeError(
-                "Cannot use cached_property instance without calling __set_name__ on it.")
-        try:
-            cache = instance.__dict__  # noqa
-        except AttributeError:  # not all objects have __dict__ (e.g. class defines slots)
-            msg = (
-                f"No '__dict__' attribute on {type(instance).__name__!r} "
-                f"instance to cache {self.attrname!r} property."
-            )
-            raise TypeError(msg) from None
-        val = cache.get(self.attrname, _NOT_FOUND)
-        if val is _NOT_FOUND:
-            with self.lock:
-                # check if another thread filled cache while we awaited lock
-                val = cache.get(self.attrname, _NOT_FOUND)
-                if val is _NOT_FOUND:
-                    val = self.func(instance)
-                    try:
-                        cache[self.attrname] = val
-                    except TypeError:
-                        msg = (
-                            f"The '__dict__' attribute on {type(instance).__name__!r} instance "
-                            f"does not support item assignment for caching {self.attrname!r} property."
-                        )
-                        raise TypeError(msg) from None
-        return val
-
-
-try:
-    from functools import cached_property  # type: ignore
-except ImportError:
-    # for python < 3.8
-    cached_property = CachedPropertyBackport  # type: ignore
-
+from functools import cache, cached_property  # noqa
 
 try:
     from numpy.typing import ArrayLike, DTypeLike  # noqa
 except ImportError:
     # numpy <1.20
     from numpy import dtype as DTypeLike  # noqa
     from numpy import ndarray as ArrayLike  # noqa
-
-
-try:
-    from functools import cache  # type: ignore
-except ImportError:
-    from functools import lru_cache as cache  # noqa

satpy/_config.py

@@ -26,20 +26,9 @@
 import tempfile
 from collections import OrderedDict
 from importlib.metadata import EntryPoint, entry_points
-from pathlib import Path
+from importlib.resources import files as impr_files
 from typing import Iterable
 
-try:
-    from importlib.resources import files as impr_files  # type: ignore
-except ImportError:
-    # Python 3.8
-    def impr_files(module_name: str) -> Path:
-        """Get path to module as a backport for Python 3.8."""
-        from importlib.resources import path as impr_path
-
-        with impr_path(module_name, "__init__.py") as pkg_init_path:
-            return pkg_init_path.parent
-
 import appdirs
 from donfig import Config
 

satpy/composites/spectral.py

@@ -114,8 +114,8 @@ class NDVIHybridGreen(SpectralBlender):
 
     This green band correction follows the same approach as the HybridGreen compositor, but with a dynamic blend
     factor `f` that depends on the pixel-level Normalized Differece Vegetation Index (NDVI). The higher the NDVI, the
-    smaller the contribution from the nir channel will be, following a liner relationship between the two ranges
-    `[ndvi_min, ndvi_max]` and `limits`.
+    smaller the contribution from the nir channel will be, following a liner (default) or non-linear relationship
+    between the two ranges `[ndvi_min, ndvi_max]` and `limits`.
 
     As an example, a new green channel using e.g. FCI data and the NDVIHybridGreen compositor can be defined like::
 
@@ -124,6 +124,7 @@ class NDVIHybridGreen(SpectralBlender):
         ndvi_min: 0.0
         ndvi_max: 1.0
         limits: [0.15, 0.05]
+        strength: 1.0
         prerequisites:
           - name: vis_05
             modifiers: [sunz_corrected, rayleigh_corrected]
@@ -138,30 +139,70 @@ class NDVIHybridGreen(SpectralBlender):
     pixels with NDVI=1.0 will be a weighted average with 5% contribution from the near-infrared
     vis_08 channel and the remaining 95% from the native green vis_05 channel. For other values of
     NDVI a linear interpolation between these values will be performed.
+
+    A strength larger or smaller than 1.0 will introduce a non-linear relationship between the two ranges
+    `[ndvi_min, ndvi_max]` and `limits`. Hence, a higher strength (> 1.0) will result in a slower transition
+    to higher/lower fractions at the NDVI extremes. Similarly, a lower strength (< 1.0) will result in a
+    faster transition to higher/lower fractions at the NDVI extremes.
     """
 
-    def __init__(self, *args, ndvi_min=0.0, ndvi_max=1.0, limits=(0.15, 0.05), **kwargs):
-        """Initialize class and set the NDVI limits and the corresponding blending fraction limits."""
+    def __init__(self, *args, ndvi_min=0.0, ndvi_max=1.0, limits=(0.15, 0.05), strength=1.0, **kwargs):
+        """Initialize class and set the NDVI limits, blending fraction limits and strength."""
+        if strength <= 0.0:
+            raise ValueError(f"Expected stength greater than 0.0, got {strength}.")
+
         self.ndvi_min = ndvi_min
         self.ndvi_max = ndvi_max
         self.limits = limits
+        self.strength = strength
         super().__init__(*args, **kwargs)
 
     def __call__(self, projectables, optional_datasets=None, **attrs):
         """Construct the hybrid green channel weighted by NDVI."""
+        LOG.info(f"Applying NDVI-weighted hybrid-green correction with limits [{self.limits[0]}, "
+                 f"{self.limits[1]}] and strength {self.strength}.")
+
         ndvi_input = self.match_data_arrays([projectables[1], projectables[2]])
 
         ndvi = (ndvi_input[1] - ndvi_input[0]) / (ndvi_input[1] + ndvi_input[0])
 
         ndvi.data = da.where(ndvi > self.ndvi_min, ndvi, self.ndvi_min)
         ndvi.data = da.where(ndvi < self.ndvi_max, ndvi, self.ndvi_max)
 
-        fraction = (ndvi - self.ndvi_min) / (self.ndvi_max - self.ndvi_min) * (self.limits[1] - self.limits[0]) \
-            + self.limits[0]
+        # Introduce non-linearity to ndvi for non-linear scaling to NIR blend fraction
+        if self.strength != 1.0:  # self._apply_strength() has no effect if strength = 1.0 -> no non-linear behaviour
+            ndvi = self._apply_strength(ndvi)
+
+        # Compute pixel-level NIR blend fractions from ndvi
+        fraction = self._compute_blend_fraction(ndvi)
+
+        # Prepare input as required by parent class (SpectralBlender)
         self.fractions = (1 - fraction, fraction)
 
         return super().__call__([projectables[0], projectables[2]], **attrs)
 
+    def _apply_strength(self, ndvi):
+        """Introduce non-linearity by applying strength factor.
+
+        The method introduces non-linearity to the ndvi for a non-linear scaling from ndvi to blend fraction in
+        `_compute_blend_fraction`. This can be used for a slower or faster transision to higher/lower fractions
+        at the ndvi extremes. If strength equals 1.0, this operation has no effect on the ndvi.
+        """
+        ndvi = ndvi ** self.strength / (ndvi ** self.strength + (1 - ndvi) ** self.strength)
+
+        return ndvi
+
+    def _compute_blend_fraction(self, ndvi):
+        """Compute pixel-level fraction of NIR signal to blend with native green signal.
+
+        This method linearly scales the input ndvi values to pixel-level blend fractions within the range
+        `[limits[0], limits[1]]` following this implementation <https://stats.stackexchange.com/a/281164>.
+        """
+        fraction = (ndvi - self.ndvi_min) / (self.ndvi_max - self.ndvi_min) * (self.limits[1] - self.limits[0]) \
+            + self.limits[0]
+
+        return fraction
+
 
 class GreenCorrector(SpectralBlender):
     """Previous class used to blend channels for green band corrections.

satpy/etc/composites/abi.yaml

@@ -537,7 +537,7 @@ composites:
           - name: C13
     standard_name: snow
 
-  night_microphysics_abi:
+  night_microphysics:
     description: >
       Nighttime Microphysics RGB, for GOESR: NASA, NOAA
     references:
@@ -555,6 +555,22 @@ composites:
       - name: C13
     standard_name: night_microphysics
 
+  night_microphysics_eum:
+    description: >
+      Nighttime Microphysics RGB following the EUMETSAT recipe
+    compositor: !!python/name:satpy.composites.GenericCompositor
+    prerequisites:
+      - compositor: !!python/name:satpy.composites.DifferenceCompositor
+        prerequisites:
+          - name: C15
+          - name: C14
+      - compositor: !!python/name:satpy.composites.DifferenceCompositor
+        prerequisites:
+          - name: C14
+          - name: C07
+      - name: C14
+    standard_name: night_microphysics
+
   fire_temperature_awips:
     description: >
       Fire Temperature RGB, for GOESR: NASA, NOAA

satpy/etc/composites/ahi.yaml

@@ -105,13 +105,15 @@ composites:
 
   ndvi_hybrid_green:
     compositor: !!python/name:satpy.composites.spectral.NDVIHybridGreen
+    limits: [0.15, 0.05]
+    strength: 3.0
     prerequisites:
     - name: B02
-      modifiers: [sunz_corrected, rayleigh_corrected]
+      modifiers: [sunz_corrected, rayleigh_corrected, sunz_reduced]
     - name: B03
-      modifiers: [sunz_corrected, rayleigh_corrected]
+      modifiers: [sunz_corrected, rayleigh_corrected, sunz_reduced]
     - name: B04
-      modifiers: [sunz_corrected]
+      modifiers: [sunz_corrected, sunz_reduced]
     standard_name: toa_bidirectional_reflectance
 
   airmass:
@@ -275,10 +277,10 @@ composites:
     compositor: !!python/name:satpy.composites.SelfSharpenedRGB
     prerequisites:
     - name: B03
-      modifiers: [sunz_corrected, rayleigh_corrected]
+      modifiers: [sunz_corrected, rayleigh_corrected, sunz_reduced]
     - name: ndvi_hybrid_green
     - name: B01
-      modifiers: [sunz_corrected, rayleigh_corrected]
+      modifiers: [sunz_corrected, rayleigh_corrected, sunz_reduced]
     high_resolution_band: red
     standard_name: true_color
 

satpy/etc/composites/ami.yaml

@@ -69,6 +69,7 @@ composites:
       currently implemented are experimental and may change in future versions of Satpy.
     compositor: !!python/name:satpy.composites.spectral.NDVIHybridGreen
     limits: [0.15, 0.05]
+    strength: 3.0
     prerequisites:
       - name: VI005
         modifiers: [sunz_corrected, rayleigh_corrected]
@@ -83,6 +84,7 @@ composites:
       Alternative to ndvi_hybrid_green, but without solar zenith or rayleigh correction.
     compositor: !!python/name:satpy.composites.spectral.NDVIHybridGreen
     limits: [0.15, 0.05]
+    strength: 3.0
     prerequisites:
       - name: VI005
       - name: VI006

satpy/etc/composites/fci.yaml

@@ -7,24 +7,26 @@ composites:
       The FCI green band at 0.51 µm deliberately misses the chlorophyll band, such that
       the signal comes from aerosols and ash rather than vegetation. An effect
       is that vegetation in a true colour RGB looks rather brown than green and barren rather red. Mixing in
-      some part of the NIR 0.8 channel reduced this effect.  Note that the fractions
+      some part of the NIR 0.8 channel reduced this effect.  Note that the fractions and non-linear strength
       currently implemented are experimental and may change in future versions of Satpy.
     compositor: !!python/name:satpy.composites.spectral.NDVIHybridGreen
     limits: [0.15, 0.05]
+    strength: 3.0
     prerequisites:
       - name: vis_05
-        modifiers: [sunz_corrected, rayleigh_corrected]
+        modifiers: [sunz_corrected, rayleigh_corrected, sunz_reduced]
       - name: vis_06
-        modifiers: [sunz_corrected, rayleigh_corrected]
+        modifiers: [sunz_corrected, rayleigh_corrected, sunz_reduced]
       - name: vis_08
-        modifiers: [sunz_corrected ]
+        modifiers: [sunz_corrected, sunz_reduced ]
     standard_name: toa_bidirectional_reflectance
 
   ndvi_hybrid_green_raw:
     description: >
       Alternative to ndvi_hybrid_green, but without solar zenith or rayleigh correction.
     compositor: !!python/name:satpy.composites.spectral.NDVIHybridGreen
     limits: [0.15, 0.05]
+    strength: 3.0
     prerequisites:
       - name: vis_05
       - name: vis_06
@@ -48,10 +50,10 @@ composites:
       of the ndvi_hybrid_green composites for details.
     prerequisites:
       - name: vis_06
-        modifiers: [sunz_corrected, rayleigh_corrected]
+        modifiers: [sunz_corrected, rayleigh_corrected, sunz_reduced]
       - name: ndvi_hybrid_green
       - name: vis_04
-        modifiers: [sunz_corrected, rayleigh_corrected]
+        modifiers: [sunz_corrected, rayleigh_corrected, sunz_reduced]
     standard_name: true_color
 
   true_color_raw_with_corrected_green: