#145028653 Alert if jobs run too often (#10)

* [Feature #145028653] Feature addition to notify the user if a check runs too often

* [Feature #145028653] Fix the failing tests

 - Modify the tests to include the newly introduced check status of "often"
 - The tests did not cover the check being up but just running too often
 - This modifies the tests to accommodate the said possibility

* [Feature #145028653] Alerts user only when job is run too often

 - Fix the bug where a job is flagged as running too often even on its very first ping.
 - Fix the previously altered tests to accommodate the "often" status.

hc/api/management/commands/sendalerts.py

@@ -21,8 +21,10 @@ def handle_many(self):
         now = timezone.now()
         going_down = query.filter(alert_after__lt=now, status="up")
         going_up = query.filter(alert_after__gt=now, status="down")
+        early_pings = query.filter(status="often")
         # Don't combine this in one query so Postgres can query using index:
-        checks = list(going_down.iterator()) + list(going_up.iterator())
+        checks = list(going_down.iterator()) + list(going_up.iterator()) \
+            + list(early_pings.iterator())
         if not checks:
             return False
 

hc/api/models.py

@@ -17,7 +17,8 @@
     ("up", "Up"),
     ("down", "Down"),
     ("new", "New"),
-    ("paused", "Paused")
+    ("paused", "Paused"),
+    ("often", "Often")
 )
 DEFAULT_TIMEOUT = td(days=1)
 DEFAULT_GRACE = td(hours=1)
@@ -85,6 +86,15 @@ def get_status(self):
             return self.status
 
         now = timezone.now()
+        if len(self.ping_set.all().order_by('-created')) > 2:
+            reversed_grace = (self.timeout/6)
+            pings_to_check = self.ping_set.all().order_by('-created')
+            previous_ping = pings_to_check[1].created
+            if self.last_ping + self.timeout + self.grace > now:
+                if (self.last_ping - previous_ping) - reversed_grace < self.timeout:
+                    return "often"
+                else:
+                    return "up"
 
         if self.last_ping + self.timeout + self.grace > now:
             return "up"

hc/api/views.py

@@ -23,9 +23,8 @@ def ping(request, code):
 
     check.n_pings = F("n_pings") + 1
     check.last_ping = timezone.now()
-    if check.status in ("new", "paused"):
+    if check.status in ("new", "paused", "often"):
         check.status = "up"
-
     check.save()
     check.refresh_from_db()
 

static/css/icomoon.css

@@ -59,4 +59,7 @@
 }
 .icon-settings:before {
   content: "\e8b8";
-}
+}
+.icon-often:before {
+  content: "\1f300";
+}

templates/front/my_checks_desktop.html

@@ -22,6 +22,9 @@
                     data-toggle="tooltip" title="Monitoring paused. Ping to resume."></span>
             {% elif check.in_grace_period %}
                 <span class="status icon-grace"></span>
+            {% elif check.get_status == "often" %}
+                <span class="status icon-often" data-toggle="tooltip"
+                  title="Ping is up but running too often."></span>
             {% elif check.get_status == "up" %}
                 <span class="status icon-up"></span>
             {% elif check.get_status == "down" %}

templates/front/my_checks_mobile.html

@@ -31,6 +31,8 @@ <h2>
                         <span class="label label-warning">LATE</span>
                     {% elif check.get_status == "up" %}
                         <span class="label label-success">UP</span>
+                    {% elif check.get_status == "often" %}
+                        <span class="label label-warning">OFTEN</span>
                     {% elif check.get_status == "down" %}
                         <span class="label label-danger">DOWN</span>
                     {% endif %}