Slow burn alert (#1120)

* added slow burn alert

* added slow burn playbook and updated index

* fixed slow burn alert duration

Author: yuriatgoogle

docs/playbooks/alerts/SlowErrorBudgetBurn.md

@@ -0,0 +1,23 @@
+# Fast Error Budget Alert
+
+This alert fires when 5% of the error budget, as determined by the availability SLO, is consumed in 6 hours.
+
+* First check if the site is up
+   * Check if https://encv.org/ loads (or appropriate domain for this environment)
+   * Check if you can login
+   * If you can, admins aren't affected
+   * If you can't login everyone is affected
+* Post in chat that you've got the alert
+   * Communicate to your team that you are actively looking at this alert to lower confusion.
+* Look at services dashboard
+   * Load https://console.cloud.google.com/monitoring/services
+   * Look at the Verification Server service and determine its health
+   * Look at the e2e-runner service request logs. This is a service executing code issue and key upload logic and may contain more information on what's going on
+   * Look for servers with elevated 5xx
+   * Look at request logs, you can navigate by hand or use the following query
+
+```
+resource.type="cloud_run_revision"
+resource.labels.service_name="e2e-runner"
+severity=ERROR
+```

docs/playbooks/index.md

@@ -11,6 +11,7 @@ production on GCP. All of our responses here are specific to GCP.
  - [ElevatedLatencyGreaterThan2s](alerts/ElevatedLatencyGreaterThan2s.md)
  - [ElevatedRateLimitedCount](alerts/ElevatedRateLimitedCount.md)
  - [FastErrorBudgetBurn](alerts/FastErrorBudgetBurn.md)
+ - [SlowErrorBudgetBurn](alerts/SlowErrorBudgetBurn.md)
  - [HostDown](alerts/HostDown.md)
  - [RealmTokenRemainingCapacityLow](alerts/RealmTokenRemainingCapacityLow.md)
  - [StackdriverExportFailed](alerts/StackdriverExportFailed.md)

terraform/alerting/alerts.tf

@@ -281,3 +281,41 @@ resource "google_monitoring_alert_policy" "fast_burn" {
     google_monitoring_slo.availability-slo
   ]
 }
+
+# slow error budget burn alert
+resource "google_monitoring_alert_policy" "slow_burn" {
+  project      = var.verification-server-project
+  display_name = "SlowErrorBudgetBurn"
+  combiner     = "OR"
+  enabled      = "true"
+  # create only if using GCLB, which means there's an SLO created
+  count = var.https-forwarding-rule == "" ? 0 : 1
+  conditions {
+    display_name = "5% burn in 6 hour"
+    condition_threshold {
+      filter     = <<-EOT
+      select_slo_burn_rate("projects/${var.verification-server-project}/services/verification-server/serviceLevelObjectives/availability-slo", "21600s")
+      EOT
+      duration   = "0s"
+      comparison = "COMPARISON_GT"
+      # burn rate = budget consumed * period / alerting window = .05 * (7 * 24 * 60)/360 = 1.4
+      threshold_value = 1.4
+      trigger {
+        count = 1
+      }
+    }
+  }
+
+  documentation {
+    content   = "${local.playbook_prefix}/SlowErrorBudgetBurn.md"
+    mime_type = "text/markdown"
+  }
+
+  notification_channels = [
+    google_monitoring_notification_channel.email.id,
+  ]
+
+  depends_on = [
+    google_monitoring_slo.availability-slo
+  ]
+}