From efabe28c4ac578eca2154453d2784332fcccae42 Mon Sep 17 00:00:00 2001 From: iback Date: Thu, 1 Aug 2024 12:38:02 +0000 Subject: [PATCH] updated to semantic class branch --- .../example_spine_matched_instance.ipynb | 28 +- panoptica/example_spine_semantic.ipynb | 121 ++----- .../example_spine_unmatched_instance.ipynb | 315 ++++++++++-------- 3 files changed, 226 insertions(+), 238 deletions(-) diff --git a/panoptica/example_spine_matched_instance.ipynb b/panoptica/example_spine_matched_instance.ipynb index e0eb96e..4fa3669 100644 --- a/panoptica/example_spine_matched_instance.ipynb +++ b/panoptica/example_spine_matched_instance.ipynb @@ -61,7 +61,7 @@ "import numpy as np\n", "from auxiliary.nifti.io import read_nifti\n", "from rich import print as pprint\n", - "from panoptica import MatchedInstancePair, Panoptic_Evaluator\n", + "from panoptica import MatchedInstancePair, Panoptica_Evaluator\n", "from panoptica.metrics import Metric" ] }, @@ -191,21 +191,20 @@ "output_type": "stream", "text": [ "Panoptic: Start Evaluation\n", - "-- Got MatchedInstancePair, will evaluate instances\n", - "-- evaluate took 2.0301928520202637 seconds to execute.\n" + "-- Got MatchedInstancePair, will evaluate instances\n" ] } ], "source": [ "sample = MatchedInstancePair(prediction_arr=pred_masks, reference_arr=ref_masks)\n", "\n", - "evaluator = Panoptic_Evaluator(\n", + "evaluator = Panoptica_Evaluator(\n", " expected_input=MatchedInstancePair,\n", " decision_metric=Metric.IOU,\n", " decision_threshold=0.5,\n", ")\n", "\n", - "result, debug_data = evaluator.evaluate(sample)" + "result, debug_data = evaluator.evaluate(sample)[\"ungrouped\"]" ] }, { @@ -237,6 +236,8 @@ "+++ GLOBAL +++\n", "Global Binary Dice (global_bin_dsc): 0.9744370224078394\n", "Global Binary Centerline Dice (global_bin_cldsc): 0.9637064011802574\n", + "Global Binary Average Symmetric Surface Distance (global_bin_assd): 0.1499152780072207\n", + "Global Binary Relative Volume Difference (global_bin_rvd): -0.0006011763989268869\n", "\n", "+++ INSTANCE +++\n", "Segmentation Quality IoU (sq): 0.8328184295330796 +- 0.15186064004517466\n", @@ -244,6 +245,7 @@ "Segmentation Quality Dsc (sq_dsc): 0.900292616009954 +- 0.10253566174957332\n", "Panoptic Quality Dsc (pq_dsc): 0.900292616009954\n", "Segmentation Quality Assd (sq_assd): 0.250331887879225 +- 0.07696680402317076\n", + "Segmentation Quality Relative Volume Difference (sq_rvd): 0.0028133049062930553 +- 0.034518928495505724\n", "\n" ] } @@ -292,9 +294,13 @@ " 'tp': 22,\n", " 'fp': 0,\n", " 'fn': 0,\n", + " 'prec': 1.0,\n", + " 'rec': 1.0,\n", " 'rq': 1.0,\n", " 'global_bin_dsc': 0.9744370224078394,\n", " 'global_bin_cldsc': 0.9637064011802574,\n", + " 'global_bin_assd': 0.1499152780072207,\n", + " 'global_bin_rvd': -0.0006011763989268869,\n", " 'sq': 0.8328184295330796,\n", " 'sq_std': 0.15186064004517466,\n", " 'pq': 0.8328184295330796,\n", @@ -302,7 +308,9 @@ " 'sq_dsc_std': 0.10253566174957332,\n", " 'pq_dsc': 0.900292616009954,\n", " 'sq_assd': 0.250331887879225,\n", - " 'sq_assd_std': 0.07696680402317076\n", + " 'sq_assd_std': 0.07696680402317076,\n", + " 'sq_rvd': 0.0028133049062930553,\n", + " 'sq_rvd_std': 0.034518928495505724\n", "}\n", "\n" ], @@ -314,9 +322,13 @@ " \u001b[32m'tp'\u001b[0m: \u001b[1;36m22\u001b[0m,\n", " \u001b[32m'fp'\u001b[0m: \u001b[1;36m0\u001b[0m,\n", " \u001b[32m'fn'\u001b[0m: \u001b[1;36m0\u001b[0m,\n", + " \u001b[32m'prec'\u001b[0m: \u001b[1;36m1.0\u001b[0m,\n", + " \u001b[32m'rec'\u001b[0m: \u001b[1;36m1.0\u001b[0m,\n", " \u001b[32m'rq'\u001b[0m: \u001b[1;36m1.0\u001b[0m,\n", " \u001b[32m'global_bin_dsc'\u001b[0m: \u001b[1;36m0.9744370224078394\u001b[0m,\n", " \u001b[32m'global_bin_cldsc'\u001b[0m: \u001b[1;36m0.9637064011802574\u001b[0m,\n", + " \u001b[32m'global_bin_assd'\u001b[0m: \u001b[1;36m0.1499152780072207\u001b[0m,\n", + " \u001b[32m'global_bin_rvd'\u001b[0m: \u001b[1;36m-0.0006011763989268869\u001b[0m,\n", " \u001b[32m'sq'\u001b[0m: \u001b[1;36m0.8328184295330796\u001b[0m,\n", " \u001b[32m'sq_std'\u001b[0m: \u001b[1;36m0.15186064004517466\u001b[0m,\n", " \u001b[32m'pq'\u001b[0m: \u001b[1;36m0.8328184295330796\u001b[0m,\n", @@ -324,7 +336,9 @@ " \u001b[32m'sq_dsc_std'\u001b[0m: \u001b[1;36m0.10253566174957332\u001b[0m,\n", " \u001b[32m'pq_dsc'\u001b[0m: \u001b[1;36m0.900292616009954\u001b[0m,\n", " \u001b[32m'sq_assd'\u001b[0m: \u001b[1;36m0.250331887879225\u001b[0m,\n", - " \u001b[32m'sq_assd_std'\u001b[0m: \u001b[1;36m0.07696680402317076\u001b[0m\n", + " \u001b[32m'sq_assd_std'\u001b[0m: \u001b[1;36m0.07696680402317076\u001b[0m,\n", + " \u001b[32m'sq_rvd'\u001b[0m: \u001b[1;36m0.0028133049062930553\u001b[0m,\n", + " \u001b[32m'sq_rvd_std'\u001b[0m: \u001b[1;36m0.034518928495505724\u001b[0m\n", "\u001b[1m}\u001b[0m\n" ] }, diff --git a/panoptica/example_spine_semantic.ipynb b/panoptica/example_spine_semantic.ipynb index adf36e3..406654a 100644 --- a/panoptica/example_spine_semantic.ipynb +++ b/panoptica/example_spine_semantic.ipynb @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -54,7 +54,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -64,7 +64,7 @@ "from panoptica import (\n", " ConnectedComponentsInstanceApproximator,\n", " NaiveThresholdMatching,\n", - " Panoptic_Evaluator,\n", + " Panoptica_Evaluator,\n", " SemanticPair,\n", ")" ] @@ -87,7 +87,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -104,96 +104,17 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 16, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
────────────────────────────────────────── Thank you for using panoptica ──────────────────────────────────────────\n",
-       "
\n" - ], - "text/plain": [ - "\u001b[92m────────────────────────────────────────── \u001b[0mThank you for using \u001b[1mpanoptica\u001b[0m\u001b[92m ──────────────────────────────────────────\u001b[0m\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
                                     Please support our development by citing                                      \n",
-       "
\n" - ], - "text/plain": [ - " Please support our development by citing \n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
                          https://github.com/BrainLesion/panoptica#citation -- Thank you!                          \n",
-       "
\n" - ], - "text/plain": [ - " \u001b[4;94mhttps://github.com/BrainLesion/panoptica#citation\u001b[0m -- Thank you! \n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n",
-       "
\n" - ], - "text/plain": [ - "\u001b[92m───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n",
-       "
\n" - ], - "text/plain": [ - "\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Panoptic: Start Evaluation\n", - "-- Got SemanticPair, will approximate instances\n", - "-- Got UnmatchedInstancePair, will match instances\n", - "-- Got MatchedInstancePair, will evaluate instances\n", - "-- evaluate took 4.104548931121826 seconds to execute.\n" - ] - } - ], + "outputs": [], "source": [ "sample = SemanticPair(pred_masks, ref_masks)\n", "\n", - "evaluator = Panoptic_Evaluator(\n", + "evaluator = Panoptica_Evaluator(\n", " expected_input=SemanticPair,\n", " instance_approximator=ConnectedComponentsInstanceApproximator(),\n", " instance_matcher=NaiveThresholdMatching(),\n", - ")\n", - "\n", - "result, debug_data = evaluator.evaluate(sample)" + ")" ] }, { @@ -206,7 +127,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 17, "metadata": {}, "outputs": [ { @@ -225,6 +146,8 @@ "+++ GLOBAL +++\n", "Global Binary Dice (global_bin_dsc): 0.9731641527805414\n", "Global Binary Centerline Dice (global_bin_cldsc): 0.9623567366028574\n", + "Global Binary Average Symmetric Surface Distance (global_bin_assd): 0.15559863304671395\n", + "Global Binary Relative Volume Difference (global_bin_rvd): 0.0021710740333888085\n", "\n", "+++ INSTANCE +++\n", "Segmentation Quality IoU (sq): 0.7940127477906024 +- 0.11547745015679488\n", @@ -232,18 +155,20 @@ "Segmentation Quality Dsc (sq_dsc): 0.8802182546605446 +- 0.07728416427007168\n", "Panoptic Quality Dsc (pq_dsc): 0.7301810521615881\n", "Segmentation Quality Assd (sq_assd): 0.2057371092494465 +- 0.1398348236766068\n", + "Segmentation Quality Relative Volume Difference (sq_rvd): 0.011340219860617232 +- 0.1217805112447998\n", "\n" ] } ], "source": [ "# print all results\n", + "result, _ = evaluator.evaluate(sample, verbose=False)[\"ungrouped\"]\n", "print(result)" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -267,7 +192,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -280,9 +205,13 @@ " 'tp': 73,\n", " 'fp': 16,\n", " 'fn': 14,\n", + " 'prec': 0.8202247191011236,\n", + " 'rec': 0.8390804597701149,\n", " 'rq': 0.8295454545454546,\n", " 'global_bin_dsc': 0.9731641527805414,\n", " 'global_bin_cldsc': 0.9623567366028574,\n", + " 'global_bin_assd': 0.15559863304671395,\n", + " 'global_bin_rvd': 0.0021710740333888085,\n", " 'sq': 0.7940127477906024,\n", " 'sq_std': 0.11547745015679488,\n", " 'pq': 0.6586696657808406,\n", @@ -290,7 +219,9 @@ " 'sq_dsc_std': 0.07728416427007168,\n", " 'pq_dsc': 0.7301810521615881,\n", " 'sq_assd': 0.2057371092494465,\n", - " 'sq_assd_std': 0.1398348236766068\n", + " 'sq_assd_std': 0.1398348236766068,\n", + " 'sq_rvd': 0.011340219860617232,\n", + " 'sq_rvd_std': 0.1217805112447998\n", "}\n", "\n" ], @@ -302,9 +233,13 @@ " \u001b[32m'tp'\u001b[0m: \u001b[1;36m73\u001b[0m,\n", " \u001b[32m'fp'\u001b[0m: \u001b[1;36m16\u001b[0m,\n", " \u001b[32m'fn'\u001b[0m: \u001b[1;36m14\u001b[0m,\n", + " \u001b[32m'prec'\u001b[0m: \u001b[1;36m0.8202247191011236\u001b[0m,\n", + " \u001b[32m'rec'\u001b[0m: \u001b[1;36m0.8390804597701149\u001b[0m,\n", " \u001b[32m'rq'\u001b[0m: \u001b[1;36m0.8295454545454546\u001b[0m,\n", " \u001b[32m'global_bin_dsc'\u001b[0m: \u001b[1;36m0.9731641527805414\u001b[0m,\n", " \u001b[32m'global_bin_cldsc'\u001b[0m: \u001b[1;36m0.9623567366028574\u001b[0m,\n", + " \u001b[32m'global_bin_assd'\u001b[0m: \u001b[1;36m0.15559863304671395\u001b[0m,\n", + " \u001b[32m'global_bin_rvd'\u001b[0m: \u001b[1;36m0.0021710740333888085\u001b[0m,\n", " \u001b[32m'sq'\u001b[0m: \u001b[1;36m0.7940127477906024\u001b[0m,\n", " \u001b[32m'sq_std'\u001b[0m: \u001b[1;36m0.11547745015679488\u001b[0m,\n", " \u001b[32m'pq'\u001b[0m: \u001b[1;36m0.6586696657808406\u001b[0m,\n", @@ -312,7 +247,9 @@ " \u001b[32m'sq_dsc_std'\u001b[0m: \u001b[1;36m0.07728416427007168\u001b[0m,\n", " \u001b[32m'pq_dsc'\u001b[0m: \u001b[1;36m0.7301810521615881\u001b[0m,\n", " \u001b[32m'sq_assd'\u001b[0m: \u001b[1;36m0.2057371092494465\u001b[0m,\n", - " \u001b[32m'sq_assd_std'\u001b[0m: \u001b[1;36m0.1398348236766068\u001b[0m\n", + " \u001b[32m'sq_assd_std'\u001b[0m: \u001b[1;36m0.1398348236766068\u001b[0m,\n", + " \u001b[32m'sq_rvd'\u001b[0m: \u001b[1;36m0.011340219860617232\u001b[0m,\n", + " \u001b[32m'sq_rvd_std'\u001b[0m: \u001b[1;36m0.1217805112447998\u001b[0m\n", "\u001b[1m}\u001b[0m\n" ] }, diff --git a/panoptica/example_spine_unmatched_instance.ipynb b/panoptica/example_spine_unmatched_instance.ipynb index 3ade848..1ea6cec 100644 --- a/panoptica/example_spine_unmatched_instance.ipynb +++ b/panoptica/example_spine_unmatched_instance.ipynb @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -54,14 +54,15 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "from auxiliary.nifti.io import read_nifti\n", "from rich import print as pprint\n", - "from panoptica import NaiveThresholdMatching, Panoptic_Evaluator, UnmatchedInstancePair" + "from panoptica import NaiveThresholdMatching, Panoptica_Evaluator, UnmatchedInstancePair\n", + "from panoptica.utils.segmentation_class import LabelGroup, SegmentationClassGroups" ] }, { @@ -83,7 +84,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 29, "metadata": {}, "outputs": [ { @@ -91,11 +92,11 @@ "text/plain": [ "(array([ 0, 2, 3, 4, 5, 6, 7, 8, 26, 102, 103, 104, 105,\n", " 106, 107, 108, 202, 203, 204, 205, 206, 207, 208], dtype=uint8),\n", - " array([ 0, 3, 4, 5, 6, 7, 8, 9, 27, 103, 104, 105, 106,\n", + " array([ 0, 3, 4, 5, 6, 7, 8, 9, 26, 103, 104, 105, 106,\n", " 107, 108, 109, 203, 204, 205, 206, 207, 208, 209], dtype=uint8))" ] }, - "execution_count": 3, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } @@ -105,6 +106,7 @@ "pred_masks = read_nifti(\"./spine_seg/unmatched_instance/pred.nii.gz\")\n", "\n", "# labels are unmatching\n", + "pred_masks[pred_masks == 27] = 26 # For later\n", "np.unique(ref_masks), np.unique(pred_masks)" ] }, @@ -117,93 +119,30 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 30, "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
────────────────────────────────────────── Thank you for using panoptica ──────────────────────────────────────────\n",
-       "
\n" - ], - "text/plain": [ - "\u001b[92m────────────────────────────────────────── \u001b[0mThank you for using \u001b[1mpanoptica\u001b[0m\u001b[92m ──────────────────────────────────────────\u001b[0m\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
                                     Please support our development by citing                                      \n",
-       "
\n" - ], - "text/plain": [ - " Please support our development by citing \n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
                          https://github.com/BrainLesion/panoptica#citation -- Thank you!                          \n",
-       "
\n" - ], - "text/plain": [ - " \u001b[4;94mhttps://github.com/BrainLesion/panoptica#citation\u001b[0m -- Thank you! \n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n",
-       "
\n" - ], - "text/plain": [ - "\u001b[92m───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n",
-       "
\n" - ], - "text/plain": [ - "\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Panoptic: Start Evaluation\n", - "-- Got UnmatchedInstancePair, will match instances\n", - "-- Got MatchedInstancePair, will evaluate instances\n", - "-- evaluate took 4.241225719451904 seconds to execute.\n" - ] - } - ], + "outputs": [], "source": [ "sample = UnmatchedInstancePair(pred_masks, ref_masks)\n", - "evaluator = Panoptic_Evaluator(\n", - " expected_input=UnmatchedInstancePair,\n", - " instance_matcher=NaiveThresholdMatching(),\n", + "\n", + "# Define (optionally) semantic groups\n", + "# This means that only instance within one group can be matched to each other\n", + "segmentation_class_groups = SegmentationClassGroups(\n", + " {\n", + " \"vertebra\": LabelGroup(list(range(1, 11))),\n", + " \"ivd\": LabelGroup(list(range(101, 111))),\n", + " \"sacrum\": ([26], True),\n", + " \"endplate\": LabelGroup(list(range(201, 211))),\n", + " }\n", ")\n", + "# In this case, the label 26 can only be matched with label 26 (thats why have to ensure above that 26 exists in both masks, otherwise they wouldn't be matched)\n", "\n", - "result, debug_data = evaluator.evaluate(sample)" + "evaluator = Panoptica_Evaluator(\n", + " expected_input=UnmatchedInstancePair,\n", + " instance_matcher=NaiveThresholdMatching(),\n", + " # If you want to use segmentation class groups, give it here as argument\n", + " segmentation_class_groups=segmentation_class_groups,\n", + ")" ] }, { @@ -216,54 +155,139 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 31, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ + "\n", + "### Group vertebra\n", + "\n", + "+++ MATCHING +++\n", + "Number of instances in reference (num_ref_instances): 7\n", + "Number of instances in prediction (num_pred_instances): 7\n", + "True Positives (tp): 7\n", + "False Positives (fp): 0\n", + "False Negatives (fn): 0\n", + "Recognition Quality / F1-Score (rq): 1.0\n", + "\n", + "+++ GLOBAL +++\n", + "Global Binary Dice (global_bin_dsc): 0.9631786034883428\n", + "Global Binary Centerline Dice (global_bin_cldsc): 0.9832727462458709\n", + "Global Binary Average Symmetric Surface Distance (global_bin_assd): 0.16583233154456756\n", + "Global Binary Relative Volume Difference (global_bin_rvd): -0.006522448092183933\n", + "\n", + "+++ INSTANCE +++\n", + "Segmentation Quality IoU (sq): 0.9259373047661901 +- 0.009654749671578153\n", + "Panoptic Quality IoU (pq): 0.9259373047661901\n", + "Segmentation Quality Dsc (sq_dsc): 0.9615183012231253 +- 0.005245540988039026\n", + "Panoptic Quality Dsc (pq_dsc): 0.9615183012231253\n", + "Segmentation Quality Assd (sq_assd): 0.16832296646947947 +- 0.01828381629759957\n", + "Segmentation Quality Relative Volume Difference (sq_rvd): -0.005930868093584259 +- 0.010871203881221219\n", + "\n", + "\n", + "### Group ivd\n", + "\n", + "+++ MATCHING +++\n", + "Number of instances in reference (num_ref_instances): 7\n", + "Number of instances in prediction (num_pred_instances): 7\n", + "True Positives (tp): 7\n", + "False Positives (fp): 0\n", + "False Negatives (fn): 0\n", + "Recognition Quality / F1-Score (rq): 1.0\n", + "\n", + "+++ GLOBAL +++\n", + "Global Binary Dice (global_bin_dsc): 0.9423566613429801\n", + "Global Binary Centerline Dice (global_bin_cldsc): 0.9785654029223958\n", + "Global Binary Average Symmetric Surface Distance (global_bin_assd): 0.2945667788167613\n", + "Global Binary Relative Volume Difference (global_bin_rvd): 0.02117012587992792\n", + "\n", + "+++ INSTANCE +++\n", + "Segmentation Quality IoU (sq): 0.8897861147389462 +- 0.029181150423413706\n", + "Panoptic Quality IoU (pq): 0.8897861147389462\n", + "Segmentation Quality Dsc (sq_dsc): 0.9414254100052913 +- 0.016436031942319355\n", + "Panoptic Quality Dsc (pq_dsc): 0.9414254100052913\n", + "Segmentation Quality Assd (sq_assd): 0.29013503272997326 +- 0.05544330133482135\n", + "Segmentation Quality Relative Volume Difference (sq_rvd): 0.020603174193257762 +- 0.03071580120223084\n", + "\n", + "\n", + "### Group sacrum\n", + "\n", + "+++ MATCHING +++\n", + "Number of instances in reference (num_ref_instances): 1\n", + "Number of instances in prediction (num_pred_instances): 1\n", + "True Positives (tp): 1\n", + "False Positives (fp): 0\n", + "False Negatives (fn): 0\n", + "Recognition Quality / F1-Score (rq): 1.0\n", + "\n", + "+++ GLOBAL +++\n", + "Global Binary Dice (global_bin_dsc): 0.9698239455931553\n", + "Global Binary Centerline Dice (global_bin_cldsc): 1.0\n", + "Global Binary Average Symmetric Surface Distance (global_bin_assd): 0.20907172118556794\n", + "Global Binary Relative Volume Difference (global_bin_rvd): -0.011061174622567414\n", + "\n", + "+++ INSTANCE +++\n", + "Segmentation Quality IoU (sq): 0.941415733208399 +- 0.0\n", + "Panoptic Quality IoU (pq): 0.941415733208399\n", + "Segmentation Quality Dsc (sq_dsc): 0.9698239455931553 +- 0.0\n", + "Panoptic Quality Dsc (pq_dsc): 0.9698239455931553\n", + "Segmentation Quality Assd (sq_assd): 0.20907172118556794 +- 0.0\n", + "Segmentation Quality Relative Volume Difference (sq_rvd): -0.011061174622567414 +- 0.0\n", + "\n", + "\n", + "### Group endplate\n", "\n", "+++ MATCHING +++\n", - "Number of instances in reference (num_ref_instances): 22\n", - "Number of instances in prediction (num_pred_instances): 22\n", - "True Positives (tp): 19\n", + "Number of instances in reference (num_ref_instances): 7\n", + "Number of instances in prediction (num_pred_instances): 7\n", + "True Positives (tp): 4\n", "False Positives (fp): 3\n", "False Negatives (fn): 3\n", - "Recognition Quality / F1-Score (rq): 0.8636363636363636\n", + "Recognition Quality / F1-Score (rq): 0.5714285714285714\n", "\n", "+++ GLOBAL +++\n", - "Global Binary Dice (global_bin_dsc): 0.9744370224078394\n", - "Global Binary Centerline Dice (global_bin_cldsc): 0.9637064011802574\n", + "Global Binary Dice (global_bin_dsc): 0.6793787581594264\n", + "Global Binary Centerline Dice (global_bin_cldsc): 0.7061452563556313\n", + "Global Binary Average Symmetric Surface Distance (global_bin_assd): 0.3688658918466393\n", + "Global Binary Relative Volume Difference (global_bin_rvd): 0.01340152790366438\n", "\n", "+++ INSTANCE +++\n", - "Segmentation Quality IoU (sq): 0.8328184295330796 +- 0.15186064004517466\n", - "Panoptic Quality IoU (pq): 0.719252280051296\n", - "Segmentation Quality Dsc (sq_dsc): 0.900292616009954 +- 0.10253566174957332\n", - "Panoptic Quality Dsc (pq_dsc): 0.7775254410995057\n", - "Segmentation Quality Assd (sq_assd): 0.250331887879225 +- 0.07696680402317076\n", + "Segmentation Quality IoU (sq): 0.54301762284604 +- 0.01014458743300687\n", + "Panoptic Quality IoU (pq): 0.31029578448345146\n", + "Segmentation Quality Dsc (sq_dsc): 0.7037824449992637 +- 0.008529812661560601\n", + "Panoptic Quality Dsc (pq_dsc): 0.40216139714243637\n", + "Segmentation Quality Assd (sq_assd): 0.33450703853088465 +- 0.010995297631511717\n", + "Segmentation Quality Relative Volume Difference (sq_rvd): -0.009548043713894769 +- 0.05397632450411714\n", "\n" ] } ], "source": [ "# print all results\n", - "print(result)" + "results = evaluator.evaluate(sample, verbose=False)\n", + "# The groups will have the names specified above\n", + "for groupname, (result, debug) in results.items():\n", + " print()\n", + " print(\"### Group\", groupname)\n", + " print(result)" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 35, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "
result.pq=0.719252280051296\n",
+       "
results['vertebra'][0].pq=0.9259373047661901\n",
        "
\n"
       ],
       "text/plain": [
-       "result.\u001b[33mpq\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.719252280051296\u001b[0m\n"
+       "results\u001b[1m[\u001b[0m\u001b[32m'vertebra'\u001b[0m\u001b[1m]\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1m]\u001b[0m.\u001b[33mpq\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.9259373047661901\u001b[0m\n"
       ]
      },
      "metadata": {},
@@ -272,12 +296,13 @@
    ],
    "source": [
     "# get specific metric, e.g. pq\n",
-    "pprint(f\"{result.pq=}\")"
+    "# Now we need to specify group first\n",
+    "pprint(f\"{results['vertebra'][0].pq=}\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 36,
    "metadata": {},
    "outputs": [
     {
@@ -285,44 +310,56 @@
       "text/html": [
        "
results dict: \n",
        "{\n",
-       "    'num_ref_instances': 22,\n",
-       "    'num_pred_instances': 22,\n",
-       "    'tp': 19,\n",
-       "    'fp': 3,\n",
-       "    'fn': 3,\n",
-       "    'rq': 0.8636363636363636,\n",
-       "    'global_bin_dsc': 0.9744370224078394,\n",
-       "    'global_bin_cldsc': 0.9637064011802574,\n",
-       "    'sq': 0.8328184295330796,\n",
-       "    'sq_std': 0.15186064004517466,\n",
-       "    'pq': 0.719252280051296,\n",
-       "    'sq_dsc': 0.900292616009954,\n",
-       "    'sq_dsc_std': 0.10253566174957332,\n",
-       "    'pq_dsc': 0.7775254410995057,\n",
-       "    'sq_assd': 0.250331887879225,\n",
-       "    'sq_assd_std': 0.07696680402317076\n",
+       "    'num_ref_instances': 7,\n",
+       "    'num_pred_instances': 7,\n",
+       "    'tp': 7,\n",
+       "    'fp': 0,\n",
+       "    'fn': 0,\n",
+       "    'prec': 1.0,\n",
+       "    'rec': 1.0,\n",
+       "    'rq': 1.0,\n",
+       "    'global_bin_dsc': 0.9631786034883428,\n",
+       "    'global_bin_cldsc': 0.9832727462458709,\n",
+       "    'global_bin_assd': 0.16583233154456756,\n",
+       "    'global_bin_rvd': -0.006522448092183933,\n",
+       "    'sq': 0.9259373047661901,\n",
+       "    'sq_std': 0.009654749671578153,\n",
+       "    'pq': 0.9259373047661901,\n",
+       "    'sq_dsc': 0.9615183012231253,\n",
+       "    'sq_dsc_std': 0.005245540988039026,\n",
+       "    'pq_dsc': 0.9615183012231253,\n",
+       "    'sq_assd': 0.16832296646947947,\n",
+       "    'sq_assd_std': 0.01828381629759957,\n",
+       "    'sq_rvd': -0.005930868093584259,\n",
+       "    'sq_rvd_std': 0.010871203881221219\n",
        "}\n",
        "
\n"
       ],
       "text/plain": [
        "results dict: \n",
        "\u001b[1m{\u001b[0m\n",
-       "    \u001b[32m'num_ref_instances'\u001b[0m: \u001b[1;36m22\u001b[0m,\n",
-       "    \u001b[32m'num_pred_instances'\u001b[0m: \u001b[1;36m22\u001b[0m,\n",
-       "    \u001b[32m'tp'\u001b[0m: \u001b[1;36m19\u001b[0m,\n",
-       "    \u001b[32m'fp'\u001b[0m: \u001b[1;36m3\u001b[0m,\n",
-       "    \u001b[32m'fn'\u001b[0m: \u001b[1;36m3\u001b[0m,\n",
-       "    \u001b[32m'rq'\u001b[0m: \u001b[1;36m0.8636363636363636\u001b[0m,\n",
-       "    \u001b[32m'global_bin_dsc'\u001b[0m: \u001b[1;36m0.9744370224078394\u001b[0m,\n",
-       "    \u001b[32m'global_bin_cldsc'\u001b[0m: \u001b[1;36m0.9637064011802574\u001b[0m,\n",
-       "    \u001b[32m'sq'\u001b[0m: \u001b[1;36m0.8328184295330796\u001b[0m,\n",
-       "    \u001b[32m'sq_std'\u001b[0m: \u001b[1;36m0.15186064004517466\u001b[0m,\n",
-       "    \u001b[32m'pq'\u001b[0m: \u001b[1;36m0.719252280051296\u001b[0m,\n",
-       "    \u001b[32m'sq_dsc'\u001b[0m: \u001b[1;36m0.900292616009954\u001b[0m,\n",
-       "    \u001b[32m'sq_dsc_std'\u001b[0m: \u001b[1;36m0.10253566174957332\u001b[0m,\n",
-       "    \u001b[32m'pq_dsc'\u001b[0m: \u001b[1;36m0.7775254410995057\u001b[0m,\n",
-       "    \u001b[32m'sq_assd'\u001b[0m: \u001b[1;36m0.250331887879225\u001b[0m,\n",
-       "    \u001b[32m'sq_assd_std'\u001b[0m: \u001b[1;36m0.07696680402317076\u001b[0m\n",
+       "    \u001b[32m'num_ref_instances'\u001b[0m: \u001b[1;36m7\u001b[0m,\n",
+       "    \u001b[32m'num_pred_instances'\u001b[0m: \u001b[1;36m7\u001b[0m,\n",
+       "    \u001b[32m'tp'\u001b[0m: \u001b[1;36m7\u001b[0m,\n",
+       "    \u001b[32m'fp'\u001b[0m: \u001b[1;36m0\u001b[0m,\n",
+       "    \u001b[32m'fn'\u001b[0m: \u001b[1;36m0\u001b[0m,\n",
+       "    \u001b[32m'prec'\u001b[0m: \u001b[1;36m1.0\u001b[0m,\n",
+       "    \u001b[32m'rec'\u001b[0m: \u001b[1;36m1.0\u001b[0m,\n",
+       "    \u001b[32m'rq'\u001b[0m: \u001b[1;36m1.0\u001b[0m,\n",
+       "    \u001b[32m'global_bin_dsc'\u001b[0m: \u001b[1;36m0.9631786034883428\u001b[0m,\n",
+       "    \u001b[32m'global_bin_cldsc'\u001b[0m: \u001b[1;36m0.9832727462458709\u001b[0m,\n",
+       "    \u001b[32m'global_bin_assd'\u001b[0m: \u001b[1;36m0.16583233154456756\u001b[0m,\n",
+       "    \u001b[32m'global_bin_rvd'\u001b[0m: \u001b[1;36m-0.006522448092183933\u001b[0m,\n",
+       "    \u001b[32m'sq'\u001b[0m: \u001b[1;36m0.9259373047661901\u001b[0m,\n",
+       "    \u001b[32m'sq_std'\u001b[0m: \u001b[1;36m0.009654749671578153\u001b[0m,\n",
+       "    \u001b[32m'pq'\u001b[0m: \u001b[1;36m0.9259373047661901\u001b[0m,\n",
+       "    \u001b[32m'sq_dsc'\u001b[0m: \u001b[1;36m0.9615183012231253\u001b[0m,\n",
+       "    \u001b[32m'sq_dsc_std'\u001b[0m: \u001b[1;36m0.005245540988039026\u001b[0m,\n",
+       "    \u001b[32m'pq_dsc'\u001b[0m: \u001b[1;36m0.9615183012231253\u001b[0m,\n",
+       "    \u001b[32m'sq_assd'\u001b[0m: \u001b[1;36m0.16832296646947947\u001b[0m,\n",
+       "    \u001b[32m'sq_assd_std'\u001b[0m: \u001b[1;36m0.01828381629759957\u001b[0m,\n",
+       "    \u001b[32m'sq_rvd'\u001b[0m: \u001b[1;36m-0.005930868093584259\u001b[0m,\n",
+       "    \u001b[32m'sq_rvd_std'\u001b[0m: \u001b[1;36m0.010871203881221219\u001b[0m\n",
        "\u001b[1m}\u001b[0m\n"
       ]
      },
@@ -332,7 +369,7 @@
    ],
    "source": [
     "# get dict for further processing, e.g. for pandas\n",
-    "pprint(\"results dict: \", result.to_dict())"
+    "pprint(\"results dict: \", results['vertebra'][0].to_dict())"
    ]
   },
   {