Add new metric "quality with no reference"

CHANGELOG.md

@@ -15,6 +15,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 -
 
 
+- Added `QualityWithNoReference` metric ([#2288](https://github.com/Lightning-AI/torchmetrics/pull/2288))
+
+
 ### Changed
 
 -

docs/source/image/quality_with_no_reference.rst

@@ -0,0 +1,21 @@
+.. customcarditem::
+   :header: Quality with No Reference
+   :image: https://pl-flash-data.s3.amazonaws.com/assets/thumbnails/image_classification.svg
+   :tags: Image
+
+.. include:: ../links.rst
+
+#########################
+Quality with No Reference
+#########################
+
+Module Interface
+________________
+
+.. autoclass:: torchmetrics.image.QualityWithNoReference
+    :exclude-members: update, compute
+
+Functional Interface
+____________________
+
+.. autofunction:: torchmetrics.functional.image.quality_with_no_reference

docs/source/links.rst

@@ -85,6 +85,7 @@
 .. _UniversalImageQualityIndex: https://ieeexplore.ieee.org/abstract/document/995823
 .. _SpectralDistortionIndex: https://www.semanticscholar.org/paper/Multispectral-and-panchromatic-data-fusion-without-Alparone-Aiazzi/b6db12e3785326577cb95fd743fecbf5bc66c7c9
 .. _SpatialDistortionIndex: https://www.semanticscholar.org/paper/Multispectral-and-panchromatic-data-fusion-without-Alparone-Aiazzi/b6db12e3785326577cb95fd743fecbf5bc66c7c9
+.. _QualityWithNoReference: https://www.semanticscholar.org/paper/Multispectral-and-panchromatic-data-fusion-without-Alparone-Aiazzi/b6db12e3785326577cb95fd743fecbf5bc66c7c9
 .. _RelativeAverageSpectralError: https://www.semanticscholar.org/paper/Data-Fusion.-Definitions-and-Architectures-Fusion-Wald/51b2b81e5124b3bb7ec53517a5dd64d8e348cadf
 .. _WMAPE: https://en.wikipedia.org/wiki/WMAPE
 .. _CER: https://rechtsprechung-im-ostseeraum.archiv.uni-greifswald.de/word-error-rate-character-error-rate-how-to-evaluate-a-model

src/torchmetrics/functional/image/__init__.py

@@ -19,6 +19,7 @@
 from torchmetrics.functional.image.perceptual_path_length import perceptual_path_length
 from torchmetrics.functional.image.psnr import peak_signal_noise_ratio
 from torchmetrics.functional.image.psnrb import peak_signal_noise_ratio_with_blocked_effect
+from torchmetrics.functional.image.qnr import quality_with_no_reference
 from torchmetrics.functional.image.rase import relative_average_spectral_error
 from torchmetrics.functional.image.rmse_sw import root_mean_squared_error_using_sliding_window
 from torchmetrics.functional.image.sam import spectral_angle_mapper
@@ -49,4 +50,5 @@
     "learned_perceptual_image_patch_similarity",
     "perceptual_path_length",
     "spatial_correlation_coefficient",
+    "quality_with_no_reference",
 ]

src/torchmetrics/functional/image/qnr.py

@@ -0,0 +1,82 @@
+# Copyright The Lightning team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional
+
+from torch import Tensor
+from typing_extensions import Literal
+
+from torchmetrics.functional.image.d_lambda import spectral_distortion_index
+from torchmetrics.functional.image.d_s import spatial_distortion_index
+from torchmetrics.utilities.imports import _TORCHVISION_AVAILABLE
+
+if not _TORCHVISION_AVAILABLE:
+    __doctest_skip__ = ["quality_with_no_reference"]
+
+
+def quality_with_no_reference(
+    preds: Tensor,
+    ms: Tensor,
+    pan: Tensor,
+    pan_lr: Optional[Tensor] = None,
+    alpha: float = 1,
+    beta: float = 1,
+    norm_order: int = 1,
+    window_size: int = 7,
+    reduction: Literal["elementwise_mean", "sum", "none"] = "elementwise_mean",
+) -> Tensor:
+    """Calculate `Quality with No Reference`_ (QualityWithNoReference_) also known as QNR.
+
+    Metric is used to compare the joint spectral and spatial distortion between two images.
+
+    Args:
+        preds: High resolution multispectral image.
+        ms: Low resolution multispectral image.
+        pan: High resolution panchromatic image.
+        pan_lr: Low resolution panchromatic image.
+        alpha: Relevance of spectral distortion.
+        beta: Relevance of spatial distortion.
+        norm_order: Order of the norm applied on the difference.
+        window_size: Window size of the filter applied to degrade the high resolution panchromatic image.
+        reduction: A method to reduce metric score over labels.
+
+            - ``'elementwise_mean'``: takes the mean (default)
+            - ``'sum'``: takes the sum
+            - ``'none'``: no reduction will be applied
+
+    Return:
+        Tensor with QualityWithNoReference score
+
+    Raises:
+        ValueError:
+            If ``alpha`` or ``beta`` is not a non-negative real number.
+
+    Example:
+        >>> import torch
+        >>> from torchmetrics.functional.image import quality_with_no_reference
+        >>> _ = torch.manual_seed(42)
+        >>> preds = torch.rand([16, 3, 32, 32])
+        >>> ms = torch.rand([16, 3, 16, 16])
+        >>> pan = torch.rand([16, 3, 32, 32])
+        >>> quality_with_no_reference(preds, ms, pan)
+        tensor(0.9694)
+
+    """
+    if not isinstance(alpha, (int, float)) or alpha < 0:
+        raise ValueError(f"Expected `alpha` to be a non-negative real number. Got alpha: {alpha}.")
+    if not isinstance(beta, (int, float)) or beta < 0:
+        raise ValueError(f"Expected `beta` to be a non-negative real number. Got beta: {beta}.")
+    d_lambda = spectral_distortion_index(preds, ms, norm_order, reduction)
+    d_s = spatial_distortion_index(preds, ms, pan, pan_lr, norm_order, window_size, reduction)
+    return (1 - d_lambda) ** alpha * (1 - d_s) ** beta

src/torchmetrics/image/__init__.py

@@ -17,6 +17,7 @@
 from torchmetrics.image.mifid import MemorizationInformedFrechetInceptionDistance
 from torchmetrics.image.psnr import PeakSignalNoiseRatio
 from torchmetrics.image.psnrb import PeakSignalNoiseRatioWithBlockedEffect
+from torchmetrics.image.qnr import QualityWithNoReference
 from torchmetrics.image.rase import RelativeAverageSpectralError
 from torchmetrics.image.rmse_sw import RootMeanSquaredErrorUsingSlidingWindow
 from torchmetrics.image.sam import SpectralAngleMapper
@@ -46,6 +47,7 @@
     "VisualInformationFidelity",
     "TotalVariation",
     "SpatialCorrelationCoefficient",
+    "QualityWithNoReference",
 ]
 
 if _TORCH_FIDELITY_AVAILABLE:

src/torchmetrics/image/qnr.py

@@ -0,0 +1,229 @@
+# Copyright The Lightning team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Dict, List, Optional, Sequence, Union
+
+from torch import Tensor
+from typing_extensions import Literal
+
+from torchmetrics.functional.image.d_lambda import _spectral_distortion_index_compute, _spectral_distortion_index_update
+from torchmetrics.functional.image.d_s import _spatial_distortion_index_compute, _spatial_distortion_index_update
+from torchmetrics.metric import Metric
+from torchmetrics.utilities import rank_zero_warn
+from torchmetrics.utilities.data import dim_zero_cat
+from torchmetrics.utilities.imports import _MATPLOTLIB_AVAILABLE, _TORCHVISION_AVAILABLE
+from torchmetrics.utilities.plot import _AX_TYPE, _PLOT_OUT_TYPE
+
+if not _MATPLOTLIB_AVAILABLE:
+    __doctest_skip__ = ["QualityWithNoReference.plot"]
+
+if not _TORCHVISION_AVAILABLE:
+    __doctest_skip__ = ["QualityWithNoReference", "QualityWithNoReference.plot"]
+
+
+class QualityWithNoReference(Metric):
+    """Compute Quality with No Reference (QualityWithNoReference_) also now as QNR.
+
+    The metric is used to compare the joint spectral and spatial distortion between two images.
+
+    As input to ``forward`` and ``update`` the metric accepts the following input
+
+    - ``preds`` (:class:`~torch.Tensor`): High resolution multispectral image of shape ``(N,C,H,W)``.
+    - ``target`` (:class:`~Dict`): A dictionary containing the following keys:
+
+      - ``ms`` (:class:`~torch.Tensor`): Low resolution multispectral image of shape ``(N,C,H',W')``.
+      - ``pan`` (:class:`~torch.Tensor`): High resolution panchromatic image of shape ``(N,C,H,W)``.
+      - ``pan_lr`` (:class:`~torch.Tensor`): (optional) Low resolution panchromatic image of shape ``(N,C,H',W')``.
+
+    where H and W must be multiple of H' and W'.
+
+    When ``pan_lr`` is ``None``, a uniform filter will be applied on ``pan`` to produce a degraded image. The degraded
+    image is then resized to match the size of ``ms`` and served as ``pan_lr`` in the calculation.
+
+    As output of `forward` and `compute` the metric returns the following output
+
+    - ``qnr`` (:class:`~torch.Tensor`): if ``reduction!='none'`` returns float scalar tensor with average QNR value
+      over sample else returns tensor of shape ``(N,)`` with QNR values per sample
+
+    Args:
+        alpha: Relevance of spectral distortion.
+        beta: Relevance of spatial distortion.
+        norm_order: Order of the norm applied on the difference.
+        window_size: Window size of the filter applied to degrade the high resolution panchromatic image.
+        reduction: a method to reduce metric score over labels.
+
+            - ``'elementwise_mean'``: takes the mean (default)
+            - ``'sum'``: takes the sum
+            - ``'none'``: no reduction will be applied
+
+        kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more info.
+
+    Example:
+        >>> import torch
+        >>> _ = torch.manual_seed(42)
+        >>> from torchmetrics.image import QualityWithNoReference
+        >>> preds = torch.rand([16, 3, 32, 32])
+        >>> target = {
+        ...     'ms': torch.rand([16, 3, 16, 16]),
+        ...     'pan': torch.rand([16, 3, 32, 32]),
+        ... }
+        >>> qnr = QualityWithNoReference()
+        >>> qnr(preds, target)
+        tensor(0.9694)
+
+    """
+
+    higher_is_better: bool = True
+    is_differentiable: bool = True
+    full_state_update: bool = False
+    plot_lower_bound: float = 0.0
+    plot_upper_bound: float = 1.0
+
+    preds: List[Tensor]
+    ms: List[Tensor]
+    pan: List[Tensor]
+    pan_lr: List[Tensor]
+
+    def __init__(
+        self,
+        alpha: float = 1,
+        beta: float = 1,
+        norm_order: int = 1,
+        window_size: int = 7,
+        reduction: Literal["elementwise_mean", "sum", "none"] = "elementwise_mean",
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(**kwargs)
+        rank_zero_warn(
+            "Metric `QualityWithNoReference` will save all targets and predictions in buffer."
+            " For large datasets this may lead to large memory footprint."
+        )
+
+        if not isinstance(alpha, (int, float)) or alpha < 0:
+            raise ValueError(f"Expected `alpha` to be a non-negative real number. Got alpha: {alpha}.")
+        self.alpha = alpha
+        if not isinstance(beta, (int, float)) or beta < 0:
+            raise ValueError(f"Expected `beta` to be a non-negative real number. Got beta: {beta}.")
+        self.beta = beta
+        if not isinstance(norm_order, int) or norm_order <= 0:
+            raise ValueError(f"Expected `norm_order` to be a positive integer. Got norm_order: {norm_order}.")
+        self.norm_order = norm_order
+        if not isinstance(window_size, int) or window_size <= 0:
+            raise ValueError(f"Expected `window_size` to be a positive integer. Got window_size: {window_size}.")
+        self.window_size = window_size
+        allowed_reductions = ("elementwise_mean", "sum", "none")
+        if reduction not in allowed_reductions:
+            raise ValueError(f"Expected argument `reduction` be one of {allowed_reductions} but got {reduction}")
+        self.reduction = reduction
+        self.add_state("preds", default=[], dist_reduce_fx="cat")
+        self.add_state("ms", default=[], dist_reduce_fx="cat")
+        self.add_state("pan", default=[], dist_reduce_fx="cat")
+        self.add_state("pan_lr", default=[], dist_reduce_fx="cat")
+
+    def update(self, preds: Tensor, target: Dict[str, Tensor]) -> None:
+        """Update state with preds and target.
+
+        Args:
+            preds: High resolution multispectral image.
+            target: A dictionary containing the following keys:
+
+                - ``'ms'``: low resolution multispectral image.
+                - ``'pan'``: high resolution panchromatic image.
+                - ``'pan_lr'``: (optional) low resolution panchromatic image.
+
+        Raises:
+            ValueError:
+                If ``target`` doesn't have ``ms`` and ``pan``.
+
+        """
+        if "ms" not in target:
+            raise ValueError(f"Expected `target` to have key `ms`. Got target: {target.keys()}.")
+        if "pan" not in target:
+            raise ValueError(f"Expected `target` to have key `pan`. Got target: {target.keys()}.")
+        ms = target["ms"]
+        pan = target["pan"]
+        pan_lr = target["pan_lr"] if "pan_lr" in target else None
+        preds, ms = _spectral_distortion_index_update(preds, ms)
+        preds, ms, pan, pan_lr = _spatial_distortion_index_update(preds, ms, pan, pan_lr)
+        self.preds.append(preds)
+        self.ms.append(target["ms"])
+        self.pan.append(target["pan"])
+        if "pan_lr" in target:
+            self.pan_lr.append(target["pan_lr"])
+
+    def compute(self) -> Tensor:
+        """Compute and returns quality with no reference."""
+        preds = dim_zero_cat(self.preds)
+        ms = dim_zero_cat(self.ms)
+        pan = dim_zero_cat(self.pan)
+        pan_lr = dim_zero_cat(self.pan_lr) if len(self.pan_lr) > 0 else None
+        d_lambda = _spectral_distortion_index_compute(preds, ms, self.norm_order, self.reduction)
+        d_s = _spatial_distortion_index_compute(
+            preds, ms, pan, pan_lr, self.norm_order, self.window_size, self.reduction
+        )
+        return (1 - d_lambda) ** self.alpha * (1 - d_s) ** self.beta
+
+    def plot(
+        self, val: Optional[Union[Tensor, Sequence[Tensor]]] = None, ax: Optional[_AX_TYPE] = None
+    ) -> _PLOT_OUT_TYPE:
+        """Plot a single or multiple values from the metric.
+
+        Args:
+            val: Either a single result from calling `metric.forward` or `metric.compute` or a list of these results.
+                If no value is provided, will automatically call `metric.compute` and plot that result.
+            ax: An matplotlib axis object. If provided will add plot to that axis
+
+        Returns:
+            Figure and Axes object
+
+        Raises:
+            ModuleNotFoundError:
+                If `matplotlib` is not installed
+
+        .. plot::
+            :scale: 75
+
+            >>> # Example plotting a single value
+            >>> import torch
+            >>> _ = torch.manual_seed(42)
+            >>> from torchmetrics.image import QualityWithNoReference
+            >>> preds = torch.rand([16, 3, 32, 32])
+            >>> target = {
+            ...     'ms': torch.rand([16, 3, 16, 16]),
+            ...     'pan': torch.rand([16, 3, 32, 32]),
+            ... }
+            >>> metric = QualityWithNoReference()
+            >>> metric.update(preds, target)
+            >>> fig_, ax_ = metric.plot()
+
+        .. plot::
+            :scale: 75
+
+            >>> # Example plotting multiple values
+            >>> import torch
+            >>> _ = torch.manual_seed(42)
+            >>> from torchmetrics.image import QualityWithNoReference
+            >>> preds = torch.rand([16, 3, 32, 32])
+            >>> target = {
+            ...     'ms': torch.rand([16, 3, 16, 16]),
+            ...     'pan': torch.rand([16, 3, 32, 32]),
+            ... }
+            >>> metric = QualityWithNoReference()
+            >>> values = [ ]
+            >>> for _ in range(10):
+            ...     values.append(metric(preds, target))
+            >>> fig_, ax_ = metric.plot(values)
+
+        """
+        return self._plot(val, ax)

tests/unittests/helpers/testers.py

@@ -426,7 +426,7 @@ def run_class_metric_test(
         """
         atol = atol or self.atol
         metric_args = metric_args or {}
-        if ddp:
+        if ddp and hasattr(pytest, "pool"):
             if sys.platform == "win32":
                 pytest.skip("DDP not supported on windows")