Merge pull request kubernetes#15240 from hongkailiu/ghproxyAlerts

Add alerts for ghproxy

prow/cluster/monitoring/mixins/prometheus/ghproxy_alerts.libsonnet

@@ -0,0 +1,44 @@
+{
+  prometheusAlerts+:: {
+    groups+: [
+      {
+        name: 'ghproxy',
+        rules: [
+          {
+            alert: 'ghproxy-status-code-abnormal-%sXX' % code_prefix,
+            // excluding 404 because it does not indicate any error in the system
+            expr: |||
+              sum(rate(github_request_duration_count{status=~"%s..", status!="404"}[5m])) / sum(rate(github_request_duration_count{status!="404"}[5m])) * 100 > 5
+            ||| % code_prefix,
+            'for': '1m',
+            labels: {
+              severity: 'slack',
+            },
+            annotations: {
+              message: 'ghproxy has {{ $value | humanize }}%% of status code %sXX for over 1 minute.' % code_prefix,
+            },
+          }
+          for code_prefix in ['4', '5']
+        ] +
+        [
+          {
+            alert: 'ghproxy-running-out-github-tokens-in-a-hour',
+            // check 30% of the capacity (5000): 1500
+            expr: |||
+              github_token_usage{job="ghproxy"} <  1500
+              and
+              predict_linear(github_token_usage{job="ghproxy"}[1h], 1 * 3600) < 0
+            |||,
+            'for': '5m',
+            labels: {
+              severity: 'slack',
+            },
+            annotations: {
+              message: 'token {{ $labels.token_hash }} will run out of API quota before the next reset.',
+            },
+          }
+        ],
+      },
+    ],
+  },
+}

prow/cluster/monitoring/mixins/prometheus/prometheus.libsonnet

@@ -1,6 +1,7 @@
 (import 'ci_absent_alerts.libsonnet') +
 (import 'prow_monitoring_absent_alerts.libsonnet') +
 (import 'configmap_alerts.libsonnet') +
+(import 'ghproxy_alerts.libsonnet') +
 (import 'hook_alert.libsonnet') +
 (import 'sinker_alerts.libsonnet') +
 (import 'tide_alerts.libsonnet')