Merge pull request #321 from flask-dashboard/thejager/workflows

Improvements to reports

flask_monitoringdashboard/core/reporting/questions/median_latency.py

@@ -0,0 +1,81 @@
+import numpy as np
+from scipy.stats import median_test
+
+from flask_monitoringdashboard.core.reporting.questions.report_question import (
+    ReportAnswer,
+    ReportQuestion,
+)
+from flask_monitoringdashboard.database import session_scope
+from flask_monitoringdashboard.database.request import get_latencies_sample
+
+
+class MedianLatencyReportAnswer(ReportAnswer):
+    def __init__(
+        self,
+        is_significant,
+        latencies_sample=None,
+        baseline_latencies_sample=None,
+        percentual_diff=None,
+        median=None,
+        baseline_median=None,
+    ):
+        super().__init__('MEDIAN_LATENCY')
+
+        self._is_significant = is_significant
+
+        self._baseline_latencies_sample = baseline_latencies_sample
+        self._latencies_sample = latencies_sample
+
+        self._percentual_diff = percentual_diff
+
+        self._baseline_median = baseline_median
+        self._median = median
+
+    def meta(self):
+        return dict(
+            latencies_samples=dict(
+                baseline=self._baseline_latencies_sample, comparison=self._latencies_sample
+            ),
+            median=self._median,
+            baseline_median=self._baseline_median,
+            percentual_diff=self._percentual_diff,
+        )
+
+    def is_significant(self):
+        return self._is_significant
+
+
+class MedianLatency(ReportQuestion):
+    def get_answer(self, endpoint, interval, baseline_interval):
+        with session_scope() as db_session:
+            latencies_sample = get_latencies_sample(db_session, endpoint.id, interval)
+            baseline_latencies_sample = get_latencies_sample(
+                db_session, endpoint.id, baseline_interval
+            )
+
+            if len(latencies_sample) == 0 or len(baseline_latencies_sample) == 0:
+                return MedianLatencyReportAnswer(
+                    is_significant=False,
+                    latencies_sample=latencies_sample,
+                    baseline_latencies_sample=baseline_latencies_sample,
+                )
+
+            median = float(np.median(latencies_sample))
+            baseline_median = float(np.median(baseline_latencies_sample))
+
+            percentual_diff = (median - baseline_median) / baseline_median * 100
+
+            _, p, _, _ = median_test(latencies_sample, baseline_latencies_sample)
+
+            is_significant = abs(float(percentual_diff)) > 0 and float(p) < 0.05
+
+            return MedianLatencyReportAnswer(
+                is_significant=is_significant,
+                percentual_diff=percentual_diff,
+                # Sample latencies
+                latencies_sample=latencies_sample,
+                baseline_latencies_sample=baseline_latencies_sample,
+                # Latency medians
+                median=median,
+                baseline_median=baseline_median,
+            )

flask_monitoringdashboard/core/reporting/questions/report_question.py

@@ -1,8 +1,7 @@
-
 from abc import ABCMeta, abstractmethod
 
 
-class Answer:
+class ReportAnswer:
     __metaclass__ = ABCMeta
 
     def __init__(self, type):
@@ -14,13 +13,14 @@ def is_significant(self):
 
     @abstractmethod
     def meta(self):
+        """
+        Should return a `dict` that contains any additional information required on the
+        frontend. This will be included in the response.
+        """
         pass
 
     def serialize(self):
-        base = dict(
-            is_significant=self.is_significant(),
-            type=self.type
-        )
+        base = dict(is_significant=self.is_significant(), type=self.type)
         base.update(self.meta())
 
         return base
@@ -30,5 +30,5 @@ class ReportQuestion:
     __metaclass__ = ABCMeta
 
     @abstractmethod
-    def get_answer(self, endpoint, comparison_interval, compared_to_interval):
+    def get_answer(self, endpoint, interval, baseline_interval):
         pass

flask_monitoringdashboard/core/reporting/questions/status_code_distribution.py

@@ -1,9 +1,12 @@
 from flask_monitoringdashboard.controllers.requests import get_status_code_frequencies_in_interval
-from flask_monitoringdashboard.core.reporting.questions.report_question import Answer, ReportQuestion
+from flask_monitoringdashboard.core.reporting.questions.report_question import (
+    ReportAnswer,
+    ReportQuestion,
+)
 from flask_monitoringdashboard.database import session_scope
 
 
-class StatusCodeDistributionAnswer(Answer):
+class StatusCodeDistributionReportAnswer(ReportAnswer):
     def __init__(self, is_significant=False, percentages=None):
         super().__init__('STATUS_CODE_DISTRIBUTION')
 
@@ -14,9 +17,7 @@ def is_significant(self):
         return self._is_significant
 
     def meta(self):
-        return dict(
-            percentages=self.percentages
-        )
+        return dict(percentages=self.percentages)
 
 
 def frequency_to_percentage(freq, total):
@@ -27,53 +28,63 @@ def frequency_to_percentage(freq, total):
 
 
 class StatusCodeDistribution(ReportQuestion):
+    MIN_NUM_REQUESTS = 30
+    MIN_PERCENTAGE_DIFF_THRESHOLD = 3
 
-    def get_answer(self, endpoint, comparison_interval, compared_to_interval):
+    def get_answer(self, endpoint, interval, baseline_interval):
         with session_scope() as db_session:
-            comparison_interval_frequencies = get_status_code_frequencies_in_interval(db_session, endpoint.id,
-                                                                                      comparison_interval.start_date(),
-                                                                                      comparison_interval.end_date())
-
+            frequencies = get_status_code_frequencies_in_interval(
+                db_session, endpoint.id, interval.start_date(), interval.end_date()
+            )
 
-            compared_to_interval_frequencies = get_status_code_frequencies_in_interval(
-                db_session, endpoint.id,
-                compared_to_interval.start_date(),
-                compared_to_interval.end_date())
+            baseline_frequencies = get_status_code_frequencies_in_interval(
+                db_session,
+                endpoint.id,
+                baseline_interval.start_date(),
+                baseline_interval.end_date(),
+            )
 
-            registered_status_codes = set(compared_to_interval_frequencies.keys()).union(
-                set(comparison_interval_frequencies.keys()))
+        # all monitored status codes in both intervals
+        all_monitored_status_codes = set(baseline_frequencies.keys()).union(set(frequencies.keys()))
 
-            total_requests_comparison_interval = sum(comparison_interval_frequencies.values())
-            total_requests_compared_to_interval = sum(compared_to_interval_frequencies.values())
+        total_requests = sum(frequencies.values())
+        total_baseline_requests = sum(baseline_frequencies.values())
 
-            if total_requests_comparison_interval == 0 or total_requests_compared_to_interval == 0:
-                return StatusCodeDistributionAnswer(is_significant=False)
+        if (
+            total_requests < self.MIN_NUM_REQUESTS
+            or total_baseline_requests < self.MIN_NUM_REQUESTS
+        ):
+            return StatusCodeDistributionReportAnswer(is_significant=False)
 
-            percentages = []
-            max_absolute_diff = 0
+        percentages = []
+        max_percentage_diff = 0
 
-            for status_code in registered_status_codes:
-                count_comparison_interval = comparison_interval_frequencies[
-                    status_code] if status_code in comparison_interval_frequencies else 0
+        for status_code in all_monitored_status_codes:
+            count = frequencies[status_code] if status_code in frequencies else 0
 
-                count_compared_to_interval = compared_to_interval_frequencies[
-                    status_code] if status_code in compared_to_interval_frequencies else 0
+            baseline_count = (
+                baseline_frequencies[status_code] if status_code in baseline_frequencies else 0
+            )
 
-                comparison_interval_percentage = frequency_to_percentage(count_comparison_interval,
-                                                                         total_requests_comparison_interval)
+            percentage = frequency_to_percentage(freq=count, total=total_requests)
+            baseline_percentage = frequency_to_percentage(
+                freq=baseline_count, total=total_baseline_requests
+            )
 
-                compared_to_interval_percentage = frequency_to_percentage(count_compared_to_interval,
-                                                                          total_requests_compared_to_interval)
+            percentage_diff = percentage - baseline_percentage
 
-                percentage_diff = comparison_interval_percentage - compared_to_interval_percentage
-
-                percentages.append(dict(
+            percentages.append(
+                dict(
                     status_code=status_code,
-                    comparison_interval=comparison_interval_percentage,
-                    compared_to_interval=compared_to_interval_percentage,
-                    percentage_diff=percentage_diff
-                ))
+                    comparison=percentage,
+                    baseline=baseline_percentage,
+                    percentage_diff=percentage_diff,
+                )
+            )
 
-                max_absolute_diff = max(max_absolute_diff, percentage_diff)
+            max_percentage_diff = max(max_percentage_diff, percentage_diff)
 
-            return StatusCodeDistributionAnswer(is_significant=max_absolute_diff > 3, percentages=percentages)
+        return StatusCodeDistributionReportAnswer(
+            is_significant=max_percentage_diff > self.MIN_PERCENTAGE_DIFF_THRESHOLD,
+            percentages=percentages,
+        )

flask_monitoringdashboard/main.py

@@ -47,6 +47,7 @@ def to_dashboard():
 def endpoint():
     # if session_scope is imported at the top of the file, the database config won't take effect
     from flask_monitoringdashboard.database import session_scope
+
     with session_scope() as db_session:
         print(db_session.bind.dialect.name)