From c8bfee19bc9f707459c1be9f57a12876d980848c Mon Sep 17 00:00:00 2001 From: lflaherty-dstl Date: Wed, 11 Sep 2019 16:21:16 +0100 Subject: [PATCH 1/3] SIAP metric output table with red and green visual --- stonesoup/metricgenerator/base.py | 21 +++ stonesoup/metricgenerator/metrictables.py | 122 ++++++++++++++++++ .../metricgenerator/tests/test_tables.py | 36 ++++++ 3 files changed, 179 insertions(+) create mode 100644 stonesoup/metricgenerator/metrictables.py create mode 100644 stonesoup/metricgenerator/tests/test_tables.py diff --git a/stonesoup/metricgenerator/base.py b/stonesoup/metricgenerator/base.py index e9ebaf73d..9893a8636 100644 --- a/stonesoup/metricgenerator/base.py +++ b/stonesoup/metricgenerator/base.py @@ -36,6 +36,27 @@ class MetricManager(Base): """ +class MetricTableGenerator(Base): + """Metric Table base class + + Takes a set of :class: `~.Metric` objects and outputs a table of the + values next to a description and target for each metric""" + + @abstractmethod + def generate_table(self, **kwargs): + """Generate table + + Parameters + ---------- + : set of :class: `~.Metric` objects + + Returns + ------- + matplotlib.Table object""" + + raise NotImplementedError + + class PlotGenerator(MetricGenerator): """PlotGenerator base class diff --git a/stonesoup/metricgenerator/metrictables.py b/stonesoup/metricgenerator/metrictables.py new file mode 100644 index 000000000..87ff57ce5 --- /dev/null +++ b/stonesoup/metricgenerator/metrictables.py @@ -0,0 +1,122 @@ +import numpy as np +import matplotlib +from matplotlib import pyplot as plt + +from .base import MetricTableGenerator +from ..base import Property + + +class RedGreenTableGenerator(MetricTableGenerator): + """Red Green Metric Table Generator class + + Takes in a set of metrics and uses known ranges and target + values to color code an output table with respect to how + well the tracker performed in relation to each metric, where + red is worse and green is better""" + + metrics = Property(set, doc="Set of metrics to put in the table") + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.ranges = dict + self.targets = dict + self.descriptions = dict + + def generate_table(self, **kwargs): + """Generate table method + + Returns a matplotlib Table of metrics with their descriptions, target + values and a coloured value cell to represent how well the tracker has + performed in relation to each specific metric (red=bad, green=good)""" + + white = (1, 1, 1) + cellText = [["Metric", "Description", "Target", "Value"]] + cellColors = [[white, white, white, white]] + + for metric in self.metrics: + # Add metric details to table row + metric_name = metric.title + description = self.descriptions[metric_name] + target = self.targets[metric_name] + value = metric.value + cellText.append([metric_name, description, target, value]) + + # Generate color for value cell based on closeness to target value + # Closeness to infinity cannot be represented as a color + if not target == np.inf: + red_value = 1 + green_value = 1 + # A value of 1 for both red & green produces yellow + + metric_range = \ + self.ranges[metric_name][1] - self.ranges[metric_name][0] + closeness = abs(value - self.targets[metric_name]) \ + * (1 / metric_range) + + if closeness > 1: # Infinite range metric exceeded bound + closeness = 1 + if closeness > 0.5: # Further away from target + green_value = closeness + elif closeness < 0.5: # Closer to target + red_value = closeness + + cellColors.append( + [white, white, white, (red_value, green_value, 0, 0.5)]) + else: + cellColors.append([white, white, white, white]) + + # "Plot" table + fig = plt.figure(figsize=(20, 1)) + ax = fig.add_subplot(1, 1, 1) + plt.axis("off") + table = matplotlib.table.table(ax, cellText, cellColors) + table.auto_set_column_width([0, 1, 2, 3]) + table.scale(1, 4) + + return table + + +class SIAPTableGenerator(RedGreenTableGenerator): + """Red Green Table Generator specifically for SIAP metrics + + Contains methods to specify the ranges, descriptions and target values + specifically for SIAP metrics""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.set_default_ranges() + self.set_default_targets() + self.set_default_descriptions() + + def set_default_ranges(self): + self.ranges = { + "SIAP C": (0, 1), + "SIAP A": (0, 10), # True maximum is infinity + # everything >10 will be treated as "worst" + "SIAP S": (0, 1), + "SIAP LT": (0, np.inf), + "SIAP LS": (0, 1) + } + + def set_default_targets(self): + self.targets = { + "SIAP C": 1, + "SIAP A": 1, + "SIAP S": 0, + "SIAP LT": np.inf, + "SIAP LS": 1 + } + + def set_default_descriptions(self): + self.descriptions = { + "SIAP C": "Completeness, the percentage of live objects with " + "tracks on them", + "SIAP A": "Ambiguity, a measure of the number of tracks assigned " + "to each true object", + "SIAP S": "Spuriousness, the percentage of tracks unassigned to " + "any object", + "SIAP LT": "1/R where R is the average number of excess tracks " + "assigned, the higher this value the better", + "SIAP LS": "The percentage of time spent tracking true objects " + "across the dataset" + } diff --git a/stonesoup/metricgenerator/tests/test_tables.py b/stonesoup/metricgenerator/tests/test_tables.py new file mode 100644 index 000000000..d156ef552 --- /dev/null +++ b/stonesoup/metricgenerator/tests/test_tables.py @@ -0,0 +1,36 @@ +import matplotlib + +from ...types.metric import TimeRangeMetric +from ...types.time import TimeRange +from ..metrictables import SIAPTableGenerator +from ..tracktotruthmetrics import SIAPMetrics + + +def test_siaptable(): + metric_generator = SIAPMetrics() + metrics = set() + # Add metrics to ensure the generator can handle a full range of values + metrics.add(TimeRangeMetric(title="SIAP C", + value=1, + time_range=TimeRange(0, 10), + generator=metric_generator)) + metrics.add(TimeRangeMetric(title="SIAP A", + value=0.65, + time_range=TimeRange(0, 10), + generator=metric_generator)) + metrics.add(TimeRangeMetric(title="SIAP S", + value=0.35, + time_range=TimeRange(0, 10), + generator=metric_generator)) + metrics.add(TimeRangeMetric(title="SIAP LT", + value=12, + time_range=TimeRange(0, 10), + generator=metric_generator)) + metrics.add(TimeRangeMetric(title="SIAP LS", + value=0, + time_range=TimeRange(0, 10), + generator=metric_generator)) + # Generate table + table_generator = SIAPTableGenerator(metrics) + table = table_generator.generate_table() + assert isinstance(table, matplotlib.table.Table()) From a13ac0000a7ef7b54d5b9435ffabe897530cd121 Mon Sep 17 00:00:00 2001 From: lflaherty-dstl Date: Wed, 11 Sep 2019 16:34:21 +0100 Subject: [PATCH 2/3] Fix test class reference to Table --- stonesoup/metricgenerator/tests/test_tables.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stonesoup/metricgenerator/tests/test_tables.py b/stonesoup/metricgenerator/tests/test_tables.py index d156ef552..36cc2ab9a 100644 --- a/stonesoup/metricgenerator/tests/test_tables.py +++ b/stonesoup/metricgenerator/tests/test_tables.py @@ -1,4 +1,4 @@ -import matplotlib +from matplotlib.table import Table from ...types.metric import TimeRangeMetric from ...types.time import TimeRange @@ -33,4 +33,4 @@ def test_siaptable(): # Generate table table_generator = SIAPTableGenerator(metrics) table = table_generator.generate_table() - assert isinstance(table, matplotlib.table.Table()) + assert isinstance(table, Table) From 33503dd63bc98a28239d00b960d03a9053417275 Mon Sep 17 00:00:00 2001 From: rjgreen Date: Mon, 18 May 2020 09:12:48 +0100 Subject: [PATCH 3/3] Inheritance addressed --- stonesoup/metricgenerator/base.py | 18 ++++++++++++++++-- stonesoup/metricgenerator/metrictables.py | 2 +- stonesoup/metricgenerator/tests/test_tables.py | 2 +- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/stonesoup/metricgenerator/base.py b/stonesoup/metricgenerator/base.py index 9893a8636..a69fba51d 100644 --- a/stonesoup/metricgenerator/base.py +++ b/stonesoup/metricgenerator/base.py @@ -36,14 +36,14 @@ class MetricManager(Base): """ -class MetricTableGenerator(Base): +class MetricTableGenerator(MetricGenerator): """Metric Table base class Takes a set of :class: `~.Metric` objects and outputs a table of the values next to a description and target for each metric""" @abstractmethod - def generate_table(self, **kwargs): + def compute_metric(self, **kwargs): """Generate table Parameters @@ -63,3 +63,17 @@ class PlotGenerator(MetricGenerator): PlotGenerators generate metrics that are visualisations. Return :class:`~.PlottingMetric` objects. """ + + @abstractmethod + def compute_metric(self, **kwargs): + """Generate table + + Parameters + ---------- + : set of :class: `~.Metric` objects + + Returns + ------- + matplotlib.Table object""" + + raise NotImplementedError diff --git a/stonesoup/metricgenerator/metrictables.py b/stonesoup/metricgenerator/metrictables.py index 87ff57ce5..876ce5409 100644 --- a/stonesoup/metricgenerator/metrictables.py +++ b/stonesoup/metricgenerator/metrictables.py @@ -22,7 +22,7 @@ def __init__(self, *args, **kwargs): self.targets = dict self.descriptions = dict - def generate_table(self, **kwargs): + def compute_metric(self, **kwargs): """Generate table method Returns a matplotlib Table of metrics with their descriptions, target diff --git a/stonesoup/metricgenerator/tests/test_tables.py b/stonesoup/metricgenerator/tests/test_tables.py index 36cc2ab9a..abfb32d93 100644 --- a/stonesoup/metricgenerator/tests/test_tables.py +++ b/stonesoup/metricgenerator/tests/test_tables.py @@ -32,5 +32,5 @@ def test_siaptable(): generator=metric_generator)) # Generate table table_generator = SIAPTableGenerator(metrics) - table = table_generator.generate_table() + table = table_generator.compute_metric() assert isinstance(table, Table)