From 8beb3025aea5ff8f215d4c9f37956b7c39d1f428 Mon Sep 17 00:00:00 2001 From: Moh-Yakoub Date: Mon, 22 Feb 2021 21:03:56 +0000 Subject: [PATCH 1/5] Implement Jaccard Index shortcut for metrics --- ignite/metrics/__init__.py | 3 +- ignite/metrics/confusion_matrix.py | 30 ++++++++++++- tests/ignite/metrics/test_confusion_matrix.py | 45 ++++++++++++++++++- 3 files changed, 75 insertions(+), 3 deletions(-) diff --git a/ignite/metrics/__init__.py b/ignite/metrics/__init__.py index df50838d7580..610036359644 100644 --- a/ignite/metrics/__init__.py +++ b/ignite/metrics/__init__.py @@ -1,6 +1,6 @@ from ignite.metrics.accumulation import Average, GeometricAverage, VariableAccumulation from ignite.metrics.accuracy import Accuracy -from ignite.metrics.confusion_matrix import ConfusionMatrix, DiceCoefficient, IoU, mIoU +from ignite.metrics.confusion_matrix import ConfusionMatrix, DiceCoefficient, IoU, JaccardIndex, mIoU from ignite.metrics.epoch_metric import EpochMetric from ignite.metrics.fbeta import Fbeta from ignite.metrics.frequency import Frequency @@ -36,6 +36,7 @@ "GeometricAverage", "IoU", "mIoU", + "JaccardIndex", "MultiLabelConfusionMatrix", "Precision", "PSNR", diff --git a/ignite/metrics/confusion_matrix.py b/ignite/metrics/confusion_matrix.py index d4fba5d4365d..060c65cb856a 100644 --- a/ignite/metrics/confusion_matrix.py +++ b/ignite/metrics/confusion_matrix.py @@ -7,7 +7,7 @@ from ignite.metrics.metric import Metric, reinit__is_reduced, sync_all_reduce from ignite.metrics.metrics_lambda import MetricsLambda -__all__ = ["ConfusionMatrix", "mIoU", "IoU", "DiceCoefficient", "cmAccuracy", "cmPrecision", "cmRecall"] +__all__ = ["ConfusionMatrix", "mIoU", "IoU", "DiceCoefficient", "cmAccuracy", "cmPrecision", "cmRecall", "JaccardIndex"] class ConfusionMatrix(Metric): @@ -323,3 +323,31 @@ def ignore_index_fn(dice_vector: torch.Tensor) -> torch.Tensor: return MetricsLambda(ignore_index_fn, dice) else: return dice + + +def JaccardIndex(cm: ConfusionMatrix, ignore_index: Optional[int] = None) -> MetricsLambda: + r"""Calculates the Jaccard Index using :class:`~ignite.metrics.ConfusionMatrix` metric. + This is the same like the IoU metric + .. math:: \text{J}(A, B) = \frac{ \lvert A \cap B \rvert }{ \lvert A \cup B \rvert } + + Args: + cm (ConfusionMatrix): instance of confusion matrix metric + ignore_index (int, optional): index to ignore, e.g. background index + + Returns: + MetricsLambda + + Examples: + + .. code-block:: python + + train_evaluator = ... + + cm = ConfusionMatrix(num_classes=num_classes) + JaccardIndex(cm, ignore_index=0).attach(train_evaluator, 'IoU') + + state = train_evaluator.run(train_dataset) + # state.metrics['JaccardIndex'] -> tensor of shape (num_classes - 1, ) + + """ + return IoU(cm, ignore_index) diff --git a/tests/ignite/metrics/test_confusion_matrix.py b/tests/ignite/metrics/test_confusion_matrix.py index 405a06b01df4..f001ceb7ce86 100644 --- a/tests/ignite/metrics/test_confusion_matrix.py +++ b/tests/ignite/metrics/test_confusion_matrix.py @@ -7,7 +7,7 @@ import ignite.distributed as idist from ignite.exceptions import NotComputableError -from ignite.metrics import ConfusionMatrix, IoU, mIoU +from ignite.metrics import ConfusionMatrix, IoU, JaccardIndex, mIoU from ignite.metrics.confusion_matrix import DiceCoefficient, cmAccuracy, cmPrecision, cmRecall torch.manual_seed(12) @@ -647,6 +647,49 @@ def _test_distrib_accumulator_device(device): ), f"{type(cm.confusion_matrix.device)}:{cm._num_correct.device} vs {type(metric_device)}:{metric_device}" +def test_jaccard_index(): + def _test(average=None): + + y_true, y_pred = get_y_true_y_pred() + th_y_true, th_y_logits = compute_th_y_true_y_logits(y_true, y_pred) + + true_res = [0, 0, 0] + for index in range(3): + bin_y_true = y_true == index + bin_y_pred = y_pred == index + intersection = bin_y_true & bin_y_pred + union = bin_y_true | bin_y_pred + true_res[index] = intersection.sum() / union.sum() + + cm = ConfusionMatrix(num_classes=3, average=average) + jaccard_index = JaccardIndex(cm) + + # Update metric + output = (th_y_logits, th_y_true) + cm.update(output) + + res = jaccard_index.compute().numpy() + + assert np.all(res == true_res) + + for ignore_index in range(3): + cm = ConfusionMatrix(num_classes=3) + jaccard_index_metric = JaccardIndex(cm, ignore_index=ignore_index) + # Update metric + output = (th_y_logits, th_y_true) + cm.update(output) + res = jaccard_index_metric.compute().numpy() + true_res_ = true_res[:ignore_index] + true_res[ignore_index + 1 :] + assert np.all(res == true_res_), f"{ignore_index}: {res} vs {true_res_}" + + _test() + _test(average="samples") + + with pytest.raises(ValueError, match=r"ConfusionMatrix should have average attribute either"): + cm = ConfusionMatrix(num_classes=3, average="precision") + JaccardIndex(cm) + + @pytest.mark.distributed @pytest.mark.skipif(not idist.has_native_dist_support, reason="Skip if no native dist support") @pytest.mark.skipif(torch.cuda.device_count() < 1, reason="Skip if no GPU") From 744f7ebbe1483e68e96ca1dce320f8861543e203 Mon Sep 17 00:00:00 2001 From: Moh-Yakoub Date: Mon, 22 Feb 2021 21:39:04 +0000 Subject: [PATCH 2/5] fix documentations --- docs/source/metrics.rst | 2 ++ ignite/metrics/confusion_matrix.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/source/metrics.rst b/docs/source/metrics.rst index 6f593be6ebec..8f98beb8aa8e 100644 --- a/docs/source/metrics.rst +++ b/docs/source/metrics.rst @@ -319,6 +319,8 @@ Complete list of metrics .. autofunction:: mIoU +.. autofunction:: JaccardIndex + .. autoclass:: Loss .. autoclass:: MeanAbsoluteError diff --git a/ignite/metrics/confusion_matrix.py b/ignite/metrics/confusion_matrix.py index 060c65cb856a..68cd9cb639c5 100644 --- a/ignite/metrics/confusion_matrix.py +++ b/ignite/metrics/confusion_matrix.py @@ -344,7 +344,7 @@ def JaccardIndex(cm: ConfusionMatrix, ignore_index: Optional[int] = None) -> Met train_evaluator = ... cm = ConfusionMatrix(num_classes=num_classes) - JaccardIndex(cm, ignore_index=0).attach(train_evaluator, 'IoU') + JaccardIndex(cm, ignore_index=0).attach(train_evaluator, 'JaccardIndex') state = train_evaluator.run(train_dataset) # state.metrics['JaccardIndex'] -> tensor of shape (num_classes - 1, ) From 2fce8b3e262dbe97ab5f6381a8bddbd25bc4080d Mon Sep 17 00:00:00 2001 From: Moh-Yakoub Date: Mon, 22 Feb 2021 23:11:06 +0000 Subject: [PATCH 3/5] fix documentation rendering --- ignite/metrics/confusion_matrix.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ignite/metrics/confusion_matrix.py b/ignite/metrics/confusion_matrix.py index 68cd9cb639c5..891feb122015 100644 --- a/ignite/metrics/confusion_matrix.py +++ b/ignite/metrics/confusion_matrix.py @@ -326,8 +326,8 @@ def ignore_index_fn(dice_vector: torch.Tensor) -> torch.Tensor: def JaccardIndex(cm: ConfusionMatrix, ignore_index: Optional[int] = None) -> MetricsLambda: - r"""Calculates the Jaccard Index using :class:`~ignite.metrics.ConfusionMatrix` metric. - This is the same like the IoU metric + r"""Calculates the Jaccard Index using :class:`~ignite.metrics.ConfusionMatrix` metric. Implementation is based on :meth:`~ignite.metrics.IoU`. + .. math:: \text{J}(A, B) = \frac{ \lvert A \cap B \rvert }{ \lvert A \cup B \rvert } Args: From 8d9c8eaeda2ee466f8934fc8bf91f3a324a96597 Mon Sep 17 00:00:00 2001 From: Moh-Yakoub Date: Mon, 22 Feb 2021 23:18:59 +0000 Subject: [PATCH 4/5] fix too long line failure --- ignite/metrics/confusion_matrix.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ignite/metrics/confusion_matrix.py b/ignite/metrics/confusion_matrix.py index 891feb122015..fc8d947b796b 100644 --- a/ignite/metrics/confusion_matrix.py +++ b/ignite/metrics/confusion_matrix.py @@ -326,7 +326,8 @@ def ignore_index_fn(dice_vector: torch.Tensor) -> torch.Tensor: def JaccardIndex(cm: ConfusionMatrix, ignore_index: Optional[int] = None) -> MetricsLambda: - r"""Calculates the Jaccard Index using :class:`~ignite.metrics.ConfusionMatrix` metric. Implementation is based on :meth:`~ignite.metrics.IoU`. + r"""Calculates the Jaccard Index using :class:`~ignite.metrics.ConfusionMatrix` metric. + Implementation is based on :meth:`~ignite.metrics.IoU`. .. math:: \text{J}(A, B) = \frac{ \lvert A \cap B \rvert }{ \lvert A \cup B \rvert } From c7d7be63f1e1f6140b46edaac93bd91857224a8c Mon Sep 17 00:00:00 2001 From: vfdev Date: Tue, 23 Feb 2021 11:27:58 +0100 Subject: [PATCH 5/5] Removed type hints in docstring --- ignite/metrics/confusion_matrix.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ignite/metrics/confusion_matrix.py b/ignite/metrics/confusion_matrix.py index fc8d947b796b..2694148ead19 100644 --- a/ignite/metrics/confusion_matrix.py +++ b/ignite/metrics/confusion_matrix.py @@ -332,8 +332,8 @@ def JaccardIndex(cm: ConfusionMatrix, ignore_index: Optional[int] = None) -> Met .. math:: \text{J}(A, B) = \frac{ \lvert A \cap B \rvert }{ \lvert A \cup B \rvert } Args: - cm (ConfusionMatrix): instance of confusion matrix metric - ignore_index (int, optional): index to ignore, e.g. background index + cm: instance of confusion matrix metric + ignore_index: index to ignore, e.g. background index Returns: MetricsLambda