ROX-16884 Add alerts when cpu/memory commitment per availability zone…

resources/prometheus/prometheus-rules.yaml

@@ -476,3 +476,109 @@ spec:
             rhacs_org_id: "{{ $labels.rhacs_org_id }}"
             rhacs_cluster_name: "{{ $labels.rhacs_cluster_name }}"
             rhacs_environment: "{{ $labels.rhacs_environment }}"
+    - name: az-resources
+      rules:
+        - record: strictly_worker_nodes
+          expr: |
+            kube_node_role{role="worker"} * on (node) (sum(kube_node_role) by (node) == 1)
+        - record: node_availability_zone
+          expr: |
+            sum(label_replace(kube_node_labels, "availability_zone", "$1", "label_failure_domain_beta_kubernetes_io_zone", "(.*)")) by (availability_zone, node) > 0
+        - record: memory_resource_requests:strictly_worker_nodes:by_availability_zone:sum
+          expr: |
+            sum(
+              sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{resource="memory",observability="",job="kube-state-metrics"}) by (node)
+              * on (node) strictly_worker_nodes
+              * on (node) group_left(availability_zone) node_availability_zone
+            ) by (availability_zone)
+        - record: memory_resource_limits:strictly_worker_nodes:by_availability_zone:sum
+          expr: |
+            sum(
+              sum(cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{resource="memory",observability="",job="kube-state-metrics"}) by (node)
+              * on (node) strictly_worker_nodes
+              * on (node) group_left(availability_zone) node_availability_zone
+            ) by (availability_zone)
+        - record: cpu_resource_requests:strictly_worker_nodes:by_availability_zone:sum
+          expr: |
+            sum(
+              sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{resource="cpu", observability="",job="kube-state-metrics"}) by (node)
+              * on (node) strictly_worker_nodes
+              * on (node) group_left(availability_zone) node_availability_zone
+            ) by (availability_zone)
+        - record: cpu_resource_limits:strictly_worker_nodes:by_availability_zone:sum
+          expr: |
+            sum(
+              sum(cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{observability="",job="kube-state-metrics"}) by (node)
+              * on (node) strictly_worker_nodes
+              * on (node) group_left(availability_zone) node_availability_zone
+            ) by (availability_zone)
+        - record: availability_zone:strictly_worker_nodes:allocatable_cpu
+          expr: |
+            sum(
+              sum(kube_node_status_allocatable{resource="cpu"}) by (node)
+              * on (node) strictly_worker_nodes
+              * on (node) group_left(availability_zone) node_availability_zone
+            ) by (availability_zone)
+        - record: availability_zone:strictly_worker_nodes:allocatable_memory
+          expr: |
+            sum(
+              sum(kube_node_status_allocatable{resource="memory"}) by (node)
+              * on (node) strictly_worker_nodes
+              * on (node) group_left(availability_zone) node_availability_zone
+            ) by (availability_zone)
+        - record: availability_zone:strictly_worker_nodes:memory_request_ratio
+          expr: |
+            memory_resource_requests:strictly_worker_nodes:by_availability_zone:sum
+            /
+            availability_zone:strictly_worker_nodes:allocatable_memory
+        - record: availability_zone:strictly_worker_nodes:cpu_request_ratio
+          expr: |
+            cpu_resource_requests:strictly_worker_nodes:by_availability_zone:sum
+            /
+            availability_zone:strictly_worker_nodes:allocatable_cpu
+        - record: availability_zone:strictly_worker_nodes:memory_limit_ratio
+          expr: |
+            memory_resource_limits:strictly_worker_nodes:by_availability_zone:sum
+            /
+            availability_zone:strictly_worker_nodes:allocatable_memory
+        - record: availability_zone:strictly_worker_nodes:cpu_limit_ratio
+          expr: |
+            cpu_resource_limits:strictly_worker_nodes:by_availability_zone:sum
+            /
+            availability_zone:strictly_worker_nodes:allocatable_cpu
+        - alert: WorkerNodesByAZMemoryQuotaOverCommit
+          expr: availability_zone:strictly_worker_nodes:memory_request_ratio > 0.8
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "There is a high risk of over-committing Memory resources on worker nodes in AZ {{ $labels.availability_zone }}."
+            description: "During the last 5 minutes, the average memory request commitment on worker nodes in AZ {{ $labels.availability_zone }} was {{ $value | humanizePercentage }}. This is above the recommended threshold of 80%."
+            # TODO(ROX-16772)
+        - alert: WorkerNodesByAZCPUQuotaOverCommit
+          expr: availability_zone:strictly_worker_nodes:cpu_request_ratio > 0.8
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "There is a high risk of over-committing CPU resources on worker nodes in AZ {{ $labels.availability_zone }}."
+            description: "During the last 5 minutes, the average CPU request commitment on worker nodes in AZ {{ $labels.availability_zone }} was {{ $value | humanizePercentage }}. This is above the recommended threshold of 80%."
+            # TODO(ROX-16772)
+        - alert: WorkerNodesByAZMemoryOverCommit
+          expr: availability_zone:strictly_worker_nodes:memory_limit_ratio > 2
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "There is a high risk of over-committing Memory resources on worker nodes in AZ {{ $labels.availability_zone }}."
+            description: "During the last 5 minutes, the average Memory limit commitment on worker nodes in AZ {{ $labels.availability_zone }} was {{ $value | humanizePercentage }}. This is above the recommended threshold of 200%."
+            # TODO(ROX-16772)
+        - alert: WorkerNodesByAZCPUOverCommit
+          expr: availability_zone:strictly_worker_nodes:cpu_limit_ratio > 2
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "There is a high risk of over-committing CPU resources on worker nodes in AZ {{ $labels.availability_zone }}."
+            description: "During the last 5 minutes, the average CPU limit commitment on worker nodes in AZ {{ $labels.availability_zone }} was {{ $value | humanizePercentage }}. This is above the recommended threshold of 200%."
+            # TODO(ROX-16772)

resources/prometheus/unit_tests/WorkerNodesByAZCPUOverCommit.yaml

@@ -0,0 +1,31 @@
+rule_files:
+  - /tmp/prometheus-rules-test.yaml
+
+evaluation_interval: 1m
+
+tests:
+  - interval: 1m
+    input_series:
+      - series: kube_node_role{node="worker-1", role="worker"}
+        values: "1"
+      - series: kube_node_labels{node="worker-1", label_failure_domain_beta_kubernetes_io_zone="us-east-1a"}
+        values: "1"
+      - series: kube_node_status_allocatable{node="worker-1", resource="cpu", job="kube-state-metrics"}
+        values: "100"
+      - series: cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits{node="worker-1", resource="cpu", job="kube-state-metrics"}
+        values: "201"
+    alert_rule_test:
+      - eval_time: 1m
+        alertname: WorkerNodesByAZCPUOverCommit
+        exp_alerts: []
+      - eval_time: 5m
+        alertname: WorkerNodesByAZCPUOverCommit
+        exp_alerts:
+          - exp_labels:
+              alertname: WorkerNodesByAZCPUOverCommit
+              availability_zone: us-east-1a
+              severity: critical
+            exp_annotations:
+              description: "During the last 5 minutes, the average CPU limit commitment on worker nodes in AZ us-east-1a was 201%. This is above the recommended threshold of 200%."
+              summary: "There is a high risk of over-committing CPU resources on worker nodes in AZ us-east-1a."
+              # TODO(ROX-16772) sop_url

resources/prometheus/unit_tests/WorkerNodesByAZCPUQuotaOverCommit.yaml

@@ -0,0 +1,31 @@
+rule_files:
+  - /tmp/prometheus-rules-test.yaml
+
+evaluation_interval: 1m
+
+tests:
+  - interval: 1m
+    input_series:
+      - series: kube_node_role{node="worker-1", role="worker"}
+        values: "1"
+      - series: kube_node_labels{node="worker-1", label_failure_domain_beta_kubernetes_io_zone="us-east-1a"}
+        values: "1"
+      - series: kube_node_status_allocatable{node="worker-1", resource="cpu", job="kube-state-metrics"}
+        values: "100"
+      - series: cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests{node="worker-1", resource="cpu", job="kube-state-metrics"}
+        values: "81"
+    alert_rule_test:
+      - eval_time: 1m
+        alertname: WorkerNodesByAZCPUQuotaOverCommit
+        exp_alerts: []
+      - eval_time: 5m
+        alertname: WorkerNodesByAZCPUQuotaOverCommit
+        exp_alerts:
+          - exp_labels:
+              alertname: WorkerNodesByAZCPUQuotaOverCommit
+              availability_zone: us-east-1a
+              severity: critical
+            exp_annotations:
+              description: "During the last 5 minutes, the average CPU request commitment on worker nodes in AZ us-east-1a was 81%. This is above the recommended threshold of 80%."
+              summary: "There is a high risk of over-committing CPU resources on worker nodes in AZ us-east-1a."
+              # TODO(ROX-16772) sop_url

resources/prometheus/unit_tests/WorkerNodesByAZMemoryOverCommit.yaml

@@ -0,0 +1,31 @@
+rule_files:
+  - /tmp/prometheus-rules-test.yaml
+
+evaluation_interval: 1m
+
+tests:
+  - interval: 1m
+    input_series:
+      - series: kube_node_role{node="worker-1", role="worker"}
+        values: "1"
+      - series: kube_node_labels{node="worker-1", label_failure_domain_beta_kubernetes_io_zone="us-east-1a"}
+        values: "1"
+      - series: kube_node_status_allocatable{node="worker-1", resource="memory", job="kube-state-metrics"}
+        values: "100"
+      - series: cluster:namespace:pod_memory:active:kube_pod_container_resource_limits{node="worker-1", resource="memory", job="kube-state-metrics"}
+        values: "201"
+    alert_rule_test:
+      - eval_time: 1m
+        alertname: WorkerNodesByAZMemoryOverCommit
+        exp_alerts: []
+      - eval_time: 5m
+        alertname: WorkerNodesByAZMemoryOverCommit
+        exp_alerts:
+          - exp_labels:
+              alertname: WorkerNodesByAZMemoryOverCommit
+              availability_zone: us-east-1a
+              severity: critical
+            exp_annotations:
+              description: "During the last 5 minutes, the average Memory limit commitment on worker nodes in AZ us-east-1a was 201%. This is above the recommended threshold of 200%."
+              summary: "There is a high risk of over-committing Memory resources on worker nodes in AZ us-east-1a."
+              # TODO(ROX-16772) sop_url

resources/prometheus/unit_tests/WorkerNodesByAZMemoryQuotaOverCommit.yaml

@@ -0,0 +1,31 @@
+rule_files:
+  - /tmp/prometheus-rules-test.yaml
+
+evaluation_interval: 1m
+
+tests:
+  - interval: 1m
+    input_series:
+      - series: kube_node_role{node="worker-1", role="worker"}
+        values: "1"
+      - series: kube_node_labels{node="worker-1", label_failure_domain_beta_kubernetes_io_zone="us-east-1a"}
+        values: "1"
+      - series: kube_node_status_allocatable{node="worker-1", resource="memory", job="kube-state-metrics"}
+        values: "100"
+      - series: cluster:namespace:pod_memory:active:kube_pod_container_resource_requests{node="worker-1", resource="memory", job="kube-state-metrics"}
+        values: "81"
+    alert_rule_test:
+      - eval_time: 1m
+        alertname: WorkerNodesByAZMemoryQuotaOverCommit
+        exp_alerts: []
+      - eval_time: 5m
+        alertname: WorkerNodesByAZMemoryQuotaOverCommit
+        exp_alerts:
+          - exp_labels:
+              alertname: WorkerNodesByAZMemoryQuotaOverCommit
+              availability_zone: us-east-1a
+              severity: critical
+            exp_annotations:
+              description: "During the last 5 minutes, the average memory request commitment on worker nodes in AZ us-east-1a was 81%. This is above the recommended threshold of 80%."
+              summary: "There is a high risk of over-committing Memory resources on worker nodes in AZ us-east-1a."
+              # TODO(ROX-16772) sop_url