Merge remote-tracking branch 'origin/bugfix/ARTESCA-3543-missing-aler…

…ts' into w/123.0/bugfix/ARTESCA-3543-missing-alerts

CHANGELOG.md

@@ -64,6 +64,11 @@
   (PR[#3710](https://github.com/scality/metalk8s/pull/3710))
 
 ## Release 2.11.3 (in development)
+### Enhancements
+
+- Add some missing alerts in the alerts hierarchy
+  (PR[#3714](https://github.com/scality/metalk8s/pull/3714))
+
 ### Bug fixes
 
 - Fix a bug during the upgrade that makes the workload plane Ingress controller

tools/lib-alert-tree/lib_alert_tree/kubernetes.py

@@ -9,7 +9,7 @@ def pod_alerts(name, severity="warning", namespace="default"):
         models.ExistingAlert(
             alertname, severity=severity, namespace=namespace, pod=name
         )
-        for alertname in ["KubePodNotReady"]
+        for alertname in ["KubePodNotReady", "KubePodCrashLooping"]
     ]
 
 
@@ -52,3 +52,13 @@ def statefulset_alerts(name, severity="warning", namespace="default"):
             "KubeStatefulSetUpdateNotRolledOut",
         ]
     ]
+
+
+def job_alerts(name, severity="warning", namespace="default"):
+    """Common alerts for Jobs."""
+    return [
+        models.ExistingAlert(
+            alertname, severity=severity, namespace=namespace, job=name
+        )
+        for alertname in ["KubeJobCompletion", "KubeJobFailed"]
+    ]

tools/lib-alert-tree/metalk8s/nodes.py

@@ -9,12 +9,14 @@
     ),
     relationship=Relationship.ANY,
     warning_children=[
+        Existing.warning("NodeFileDescriptorLimit"),
         Existing.warning("NodeFilesystemAlmostOutOfSpace"),
         Existing.warning("NodeFilesystemAlmostOutOfFiles"),
         Existing.warning("NodeFilesystemFilesFillingUp"),
         Existing.warning("NodeFilesystemSpaceFillingUp"),
     ],
     critical_children=[
+        Existing.critical("NodeFileDescriptorLimit"),
         Existing.critical("NodeFilesystemAlmostOutOfSpace"),
         Existing.critical("NodeFilesystemAlmostOutOfFiles"),
         Existing.critical("NodeFilesystemFilesFillingUp"),

tools/lib-alert-tree/metalk8s/platform/core.py

@@ -9,6 +9,7 @@
     relationship=Relationship.ANY,
     warning_children=[
         Existing.warning("KubeAPIErrorBudgetBurn"),
+        Existing.warning("KubeAPITerminatedRequests"),
         Existing.warning("etcdHighNumberOfFailedGRPCRequests"),
         Existing.warning("etcdHTTPRequestsSlow"),
         Existing.warning("etcdHighCommitDurations"),

tools/lib-alert-tree/metalk8s/platform/observability.py

@@ -12,6 +12,7 @@
     summary_name="The monitoring service",
     relationship=Relationship.ANY,
     warning_children=[
+        Existing.warning("PrometheusLabelLimitHit"),
         Existing.warning("PrometheusTargetLimitHit"),
         Existing.warning("PrometheusTSDBReloadsFailing"),
         Existing.warning("PrometheusTSDBCompactionsFailing"),
@@ -47,11 +48,14 @@
         ),
     ],
     critical_children=[
+        Existing.critical("KubeStateMetricsShardingMismatch"),
+        Existing.critical("KubeStateMetricsShardsMissing"),
         Existing.critical("PrometheusRuleFailures"),
         Existing.critical("PrometheusRemoteWriteBehind"),
         Existing.critical("PrometheusRemoteStorageFailures"),
         Existing.critical("PrometheusErrorSendingAlertsToAnyAlertmanager"),
         Existing.critical("PrometheusBadConfig"),
+        Existing.critical("PrometheusTargetSyncFailure"),
     ],
     duration="1m",
 )
@@ -61,13 +65,18 @@
     summary_name="The alerting service",
     relationship=Relationship.ANY,
     warning_children=[
+        Existing.warning("AlertmanagerClusterFailedToSendAlerts"),
+        Existing.warning("AlertmanagerFailedToSendAlerts"),
         *statefulset_alerts(
             "alertmanager-prometheus-operator-alertmanager",
             severity="warning",
             namespace="metalk8s-monitoring",
         ),
     ],
     critical_children=[
+        Existing.critical("AlertmanagerClusterCrashlooping"),
+        Existing.critical("AlertmanagerClusterDown"),
+        Existing.critical("AlertmanagerClusterFailedToSendAlerts"),
         Existing.critical("AlertmanagerConfigInconsistent"),
         Existing.critical("AlertmanagerMembersInconsistent"),
         Existing.critical("AlertmanagerFailedReload"),

tools/lib-alert-tree/tests/test_kubernetes.py

@@ -49,6 +49,7 @@ def test_deployment_alerts():
 
 def test_pod_alerts():
     """Check Pod alerts."""
+    # pylint: disable=line-too-long
     test = D.warning(
         "test",
         children=kubernetes.pod_alerts("my-pod", namespace="my-ns"),
@@ -59,6 +60,7 @@ def test_pod_alerts():
         == textwrap.dedent(
             """
             test{severity='warning'}
+            ├── KubePodCrashLooping{namespace=~'my-ns', pod=~'my-pod', severity='warning'}
             └── KubePodNotReady{namespace=~'my-ns', pod=~'my-pod', severity='warning'}
             """
         ).lstrip()
@@ -84,3 +86,22 @@ def test_statefulset_alerts():
             """
         ).lstrip()
     )
+
+
+def test_job_alerts():
+    """Check Job alerts."""
+    test = D.warning(
+        "test",
+        children=kubernetes.job_alerts("my-job", namespace="my-ns"),
+        relationship=Relationship.ANY,
+    )
+    assert (
+        test.build_tree().show(stdout=False)
+        == textwrap.dedent(
+            """
+            test{severity='warning'}
+            ├── KubeJobCompletion{job=~'my-job', namespace=~'my-ns', severity='warning'}
+            └── KubeJobFailed{job=~'my-job', namespace=~'my-ns', severity='warning'}
+            """
+        ).lstrip()
+    )