From 9e95fac98e3fc8f1f7639fe4b8619e1182bca491 Mon Sep 17 00:00:00 2001 From: Simon Brugman Date: Wed, 28 Oct 2020 22:58:27 +0100 Subject: [PATCH] Traffic light overview (#62) * Traffic light overview plot Other changes: - suppress "matplotlib backend" verbose warning - click on "popmon report" also scrolls to top - fix example comment --- README.rst | 2 +- popmon/pipeline/report_pipelines.py | 13 +- popmon/version.py | 4 +- popmon/visualization/__init__.py | 10 +- popmon/visualization/backend.py | 9 +- popmon/visualization/section_generator.py | 47 +--- .../templates/assets/css/custom-style.css | 4 + .../templates/assets/js/custom-script.js | 7 +- popmon/visualization/templates/card.html | 2 +- popmon/visualization/templates/header.html | 2 +- .../traffic_light_section_generator.py | 239 ++++++++++++++++++ popmon/visualization/utils.py | 54 +++- 12 files changed, 337 insertions(+), 56 deletions(-) create mode 100644 popmon/visualization/traffic_light_section_generator.py diff --git a/README.rst b/README.rst index 588cd0e5..0c826865 100644 --- a/README.rst +++ b/README.rst @@ -94,7 +94,7 @@ As a quick example, you can do: # to show the output of the report in a Jupyter notebook you can simply run: report - # or save the report to file and open in a browser + # or save the report to file report.to_file("monitoring_report.html") To specify your own binning specifications and features you want to report on, you do: diff --git a/popmon/pipeline/report_pipelines.py b/popmon/pipeline/report_pipelines.py index a042b90b..483bcfdb 100644 --- a/popmon/pipeline/report_pipelines.py +++ b/popmon/pipeline/report_pipelines.py @@ -29,7 +29,12 @@ metrics_rolling_reference, metrics_self_reference, ) -from ..visualization import HistogramSection, ReportGenerator, SectionGenerator +from ..visualization import ( + HistogramSection, + ReportGenerator, + SectionGenerator, + TrafficLightSectionGenerator, +) def self_reference( @@ -338,10 +343,8 @@ def sg_kws(read_key): ignore_stat_endswith=["_mean", "_std", "_pull"], **sg_kws("comparisons"), ), - SectionGenerator( - section_name=traffic_lights_section, - tl_section=True, - **sg_kws("traffic_lights"), + TrafficLightSectionGenerator( + section_name=traffic_lights_section, **sg_kws("traffic_lights") ), SectionGenerator(section_name=alerts_section, **sg_kws("alerts")), HistogramSection( diff --git a/popmon/version.py b/popmon/version.py index 7c15638b..82742b9e 100644 --- a/popmon/version.py +++ b/popmon/version.py @@ -1,6 +1,6 @@ """THIS FILE IS AUTO-GENERATED BY SETUP.PY.""" name = "popmon" -version = "0.3.8" -full_version = "0.3.8" +version = "0.3.9" +full_version = "0.3.9" release = True diff --git a/popmon/visualization/__init__.py b/popmon/visualization/__init__.py index 52502ff2..379b81b8 100644 --- a/popmon/visualization/__init__.py +++ b/popmon/visualization/__init__.py @@ -23,6 +23,9 @@ from popmon.visualization.histogram_section import HistogramSection from popmon.visualization.report_generator import ReportGenerator from popmon.visualization.section_generator import SectionGenerator +from popmon.visualization.traffic_light_section_generator import ( + TrafficLightSectionGenerator, +) # set matplotlib backend to batchmode when running in shell # need to do this *before* matplotlib.pyplot gets imported @@ -31,4 +34,9 @@ set_matplotlib_backend() -__all__ = ["SectionGenerator", "HistogramSection", "ReportGenerator"] +__all__ = [ + "SectionGenerator", + "HistogramSection", + "ReportGenerator", + "TrafficLightSectionGenerator", +] diff --git a/popmon/visualization/backend.py b/popmon/visualization/backend.py index 3c6e50fa..3f06b172 100644 --- a/popmon/visualization/backend.py +++ b/popmon/visualization/backend.py @@ -107,10 +107,11 @@ def set_matplotlib_backend(backend=None, batch=None, silent=True): raise RuntimeError( "Cannot set Matplotlib backend: pyplot module already loaded." ) - else: - logger.warning( - "Cannot set Matplotlib backend: pyplot module already loaded." - ) + # Warning is too verbose + # else: + # logger.warning( + # "Cannot set Matplotlib backend: pyplot module already loaded." + # ) return # set matplotlib backend diff --git a/popmon/visualization/section_generator.py b/popmon/visualization/section_generator.py index 81b3cd5a..dda21ede 100644 --- a/popmon/visualization/section_generator.py +++ b/popmon/visualization/section_generator.py @@ -28,11 +28,11 @@ from ..base import Module from ..config import get_stat_description -from ..visualization.utils import plot_bars_b64, plot_traffic_lights_b64 +from ..visualization.utils import _prune, plot_bars_b64 class SectionGenerator(Module): - """This module takes the time-series data of already computed statistcs, plots the data and + """This module takes the time-series data of already computed statistics, plots the data and combines all the plots into a list which is stored together with the section name in a dictionary which later will be used for the report generation. """ @@ -55,7 +55,6 @@ def __init__( skip_empty_plots=True, description="", show_stats=None, - tl_section=False, ): """Initialize an instance of SectionGenerator. @@ -75,7 +74,6 @@ def __init__( :param bool skip_empty_plots: if false, also show empty plots in report with only nans or zeroes (optional) :param str description: description of the section. default is empty (optional) :param list show_stats: list of statistic name patterns to show in the report. If None, show all (optional) - :param bool tl_section: whether to use traffic light plotting or not """ super().__init__() self.read_key = read_key @@ -94,7 +92,6 @@ def __init__( self.skip_empty_plots = skip_empty_plots self.description = description self.show_stats = show_stats - self.tl_section = tl_section def transform(self, datastore): data_obj = self.get_datastore_object(datastore, self.read_key, dtype=dict) @@ -158,7 +155,6 @@ def short_date(date): self.skip_first_n, self.skip_last_n, self.skip_empty_plots, - self.tl_section, ) for metric in metrics ) @@ -196,7 +192,6 @@ def _plot_metric( skip_first_n, skip_last_n, skip_empty, - tl_section, ): """Split off plot histogram generation to allow for parallel processing""" # pick up static traffic light boundaries @@ -218,34 +213,12 @@ def _plot_metric( values = _prune(values, last_n, skip_first_n, skip_last_n) # make plot. note: slow! - if tl_section: - plot = plot_traffic_lights_b64( - data=np.array(values), labels=dates, skip_empty=skip_empty - ) - else: - plot = plot_bars_b64( - data=np.array(values), - labels=dates, - ylim=True, - bounds=bounds, - skip_empty=skip_empty, - ) - return dict(name=metric, description=get_stat_description(metric), plot=plot) - - -def _prune(values, last_n=0, skip_first_n=0, skip_last_n=0): - """inline function to select first or last items of input list + plot = plot_bars_b64( + data=np.array(values), + labels=dates, + ylim=True, + bounds=bounds, + skip_empty=skip_empty, + ) - :param values: input list to select from - :param int last_n: select last 'n' items of values. default is 0. - :param int skip_first_n: skip first n items of values. default is 0. last_n takes precedence. - :param int skip_last_n: in plot skip last 'n' periods. last_n takes precedence (optional) - :return: list of selected values - """ - if last_n > 0: - return values[-last_n:] - if skip_first_n > 0: - values = values[skip_first_n:] - if skip_last_n > 0: - values = values[:-skip_last_n] - return values + return dict(name=metric, description=get_stat_description(metric), plot=plot) diff --git a/popmon/visualization/templates/assets/css/custom-style.css b/popmon/visualization/templates/assets/css/custom-style.css index a5fc19ea..381e1e8d 100644 --- a/popmon/visualization/templates/assets/css/custom-style.css +++ b/popmon/visualization/templates/assets/css/custom-style.css @@ -49,4 +49,8 @@ section h2 { .feature-select button { font-weight: 300 +} + +.card-image-top { + padding-bottom: 7px; } \ No newline at end of file diff --git a/popmon/visualization/templates/assets/js/custom-script.js b/popmon/visualization/templates/assets/js/custom-script.js index 4b3f2f86..fdbe64c3 100644 --- a/popmon/visualization/templates/assets/js/custom-script.js +++ b/popmon/visualization/templates/assets/js/custom-script.js @@ -10,7 +10,7 @@ $(document).on("click", "button.dropdown-item", function() { obj.parent().siblings("button").text("Feature: " + obj.text()) }); // making navigation work: after clicking a nav link scrolling to the corresponding section's position -$(document).on("click", "a.nav-link", function(e) { +$(document).on("click", "a.nav-link,a.navbar-brand", function(e) { e.preventDefault(); obj = $(this) $([document.documentElement, document.body]).animate({ @@ -18,10 +18,13 @@ $(document).on("click", "a.nav-link", function(e) { }, 1000); }); // automatic insertion of navigation links based on section titles -$('section').each(function(){ +$('section').each(function(i, el){ title = $(this).attr("data-section-title"); code = '
  • ' + title + '
    • ' $("ul#navigation-sections").append(code); + if ( i === 0) { + $("a.navbar-brand").attr('data-scroll-to-section', title); + } }); $('#myModal').on('shown.bs.modal', function () { diff --git a/popmon/visualization/templates/card.html b/popmon/visualization/templates/card.html index aeeb65b2..143f016d 100644 --- a/popmon/visualization/templates/card.html +++ b/popmon/visualization/templates/card.html @@ -1,4 +1,4 @@ -
    +

    {{metric.name}}

    diff --git a/popmon/visualization/templates/header.html b/popmon/visualization/templates/header.html index b764f285..50d85a31 100644 --- a/popmon/visualization/templates/header.html +++ b/popmon/visualization/templates/header.html @@ -1,7 +1,7 @@