diff --git a/charts/node-exporter-mixin/Chart.yaml b/charts/node-exporter-mixin/Chart.yaml index 0caab11d..58049173 100644 --- a/charts/node-exporter-mixin/Chart.yaml +++ b/charts/node-exporter-mixin/Chart.yaml @@ -32,12 +32,12 @@ keywords: # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.3.0 +version: 1.4.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. -appVersion: 1.4.0 +appVersion: 1.7.0 maintainers: - name: nlamirault @@ -61,4 +61,4 @@ annotations: url: https://keybase.io/nlamirault/pgp_keys.asc artifacthub.io/changes: | - kind: changed - description: node-exporter-mixin v1.5.0 + description: node-exporter-mixin v1.7.0 diff --git a/charts/node-exporter-mixin/templates/alerts.yaml b/charts/node-exporter-mixin/templates/alerts.yaml index 5a3f4dd9..f95d3433 100644 --- a/charts/node-exporter-mixin/templates/alerts.yaml +++ b/charts/node-exporter-mixin/templates/alerts.yaml @@ -17,7 +17,7 @@ spec: rules: - alert: NodeFilesystemSpaceFillingUp annotations: - description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up. + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up. summary: Filesystem is predicted to run out of space within the next 24 hours. expr: | ( @@ -32,7 +32,7 @@ spec: severity: warning - alert: NodeFilesystemSpaceFillingUp annotations: - description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up fast. + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left and is filling up fast. summary: Filesystem is predicted to run out of space within the next 4 hours. expr: | ( @@ -47,7 +47,7 @@ spec: severity: critical - alert: NodeFilesystemAlmostOutOfSpace annotations: - description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. summary: Filesystem has less than 20% space left. expr: | ( @@ -60,7 +60,7 @@ spec: severity: warning - alert: NodeFilesystemAlmostOutOfSpace annotations: - description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available space left. summary: Filesystem has less than 10% space left. expr: | ( @@ -73,7 +73,7 @@ spec: severity: critical - alert: NodeFilesystemFilesFillingUp annotations: - description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up. + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up. summary: Filesystem is predicted to run out of inodes within the next 24 hours. expr: | ( @@ -88,7 +88,7 @@ spec: severity: warning - alert: NodeFilesystemFilesFillingUp annotations: - description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up fast. + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left and is filling up fast. summary: Filesystem is predicted to run out of inodes within the next 4 hours. expr: | ( @@ -103,7 +103,7 @@ spec: severity: critical - alert: NodeFilesystemAlmostOutOfFiles annotations: - description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. summary: Filesystem has less than 5% inodes left. expr: | ( @@ -116,7 +116,7 @@ spec: severity: warning - alert: NodeFilesystemAlmostOutOfFiles annotations: - description: Filesystem on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. + description: Filesystem on {{`{{`}} $labels.device {{`}}`}}, mounted on {{`{{`}} $labels.mountpoint {{`}}`}}, at {{`{{`}} $labels.instance {{`}}`}} has only {{`{{`}} printf "%.2f" $value {{`}}`}}% available inodes left. summary: Filesystem has less than 3% inodes left. expr: | ( @@ -132,7 +132,7 @@ spec: description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} receive errors in the last two minutes.' summary: Network interface is reporting many receive errors. expr: | - rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) > 0.01 + rate(node_network_receive_errs_total{job="node-exporter"}[2m]) / rate(node_network_receive_packets_total{job="node-exporter"}[2m]) > 0.01 for: 1h labels: severity: warning @@ -141,7 +141,7 @@ spec: description: '{{`{{`}} $labels.instance {{`}}`}} interface {{`{{`}} $labels.device {{`}}`}} has encountered {{`{{`}} printf "%.0f" $value {{`}}`}} transmit errors in the last two minutes.' summary: Network interface is reporting many transmit errors. expr: | - rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) > 0.01 + rate(node_network_transmit_errs_total{job="node-exporter"}[2m]) / rate(node_network_transmit_packets_total{job="node-exporter"}[2m]) > 0.01 for: 1h labels: severity: warning @@ -150,12 +150,12 @@ spec: description: '{{`{{`}} $value | humanizePercentage {{`}}`}} of conntrack entries are used.' summary: Number of conntrack are getting close to the limit. expr: | - (node_nf_conntrack_entries / node_nf_conntrack_entries_limit) > 0.75 + (node_nf_conntrack_entries{job="node-exporter"} / node_nf_conntrack_entries_limit) > 0.75 labels: severity: warning - alert: NodeTextFileCollectorScrapeError annotations: - description: Node Exporter text file collector failed to scrape. + description: Node Exporter text file collector on {{`{{`}} $labels.instance {{`}}`}} failed to scrape. summary: Node Exporter text file collector failed to scrape. expr: | node_textfile_scrape_error{job="node-exporter"} == 1 @@ -163,7 +163,7 @@ spec: severity: warning - alert: NodeClockSkewDetected annotations: - description: Clock on {{`{{`}} $labels.instance {{`}}`}} is out of sync by more than 300s. Ensure NTP is configured correctly on this host. + description: Clock at {{`{{`}} $labels.instance {{`}}`}} is out of sync by more than 0.05s. Ensure NTP is configured correctly on this host. summary: Clock skew detected. expr: | ( @@ -182,7 +182,7 @@ spec: severity: warning - alert: NodeClockNotSynchronising annotations: - description: Clock on {{`{{`}} $labels.instance {{`}}`}} is not synchronising. Ensure NTP is configured on this host. + description: Clock at {{`{{`}} $labels.instance {{`}}`}} is not synchronising. Ensure NTP is configured on this host. summary: Clock not synchronising. expr: | min_over_time(node_timex_sync_status{job="node-exporter"}[5m]) == 0 @@ -193,8 +193,8 @@ spec: severity: warning - alert: NodeRAIDDegraded annotations: - description: RAID array '{{`{{`}} $labels.device {{`}}`}}' on {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically. - summary: RAID Array is degraded + description: RAID array '{{`{{`}} $labels.device {{`}}`}}' at {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically. + summary: RAID Array is degraded. expr: | node_md_disks_required{job="node-exporter",device!=""} - ignoring (state) (node_md_disks{state="active",job="node-exporter",device!=""}) > 0 for: 15m @@ -202,8 +202,8 @@ spec: severity: critical - alert: NodeRAIDDiskFailure annotations: - description: At least one device in RAID array on {{`{{`}} $labels.instance {{`}}`}} failed. Array '{{`{{`}} $labels.device {{`}}`}}' needs attention and possibly a disk swap. - summary: Failed device in RAID array + description: At least one device in RAID array at {{`{{`}} $labels.instance {{`}}`}} failed. Array '{{`{{`}} $labels.device {{`}}`}}' needs attention and possibly a disk swap. + summary: Failed device in RAID array. expr: | node_md_disks{state="failed",job="node-exporter",device!=""} > 0 labels: @@ -230,3 +230,75 @@ spec: for: 15m labels: severity: critical + - alert: NodeCPUHighUsage + annotations: + description: | + CPU usage at {{`{{`}} $labels.instance {{`}}`}} has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + summary: High CPU usage. + expr: | + sum without(mode) (avg without (cpu) (rate(node_cpu_seconds_total{job="node-exporter", mode!="idle"}[2m]))) * 100 > 90 + for: 15m + labels: + severity: info + - alert: NodeSystemSaturation + annotations: + description: | + System load per core at {{`{{`}} $labels.instance {{`}}`}} has been above 2 for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + This might indicate this instance resources saturation and can cause it becoming unresponsive. + summary: System saturated, load per core is very high. + expr: | + node_load1{job="node-exporter"} + / count without (cpu, mode) (node_cpu_seconds_total{job="node-exporter", mode="idle"}) > 2 + for: 15m + labels: + severity: warning + - alert: NodeMemoryMajorPagesFaults + annotations: + description: | + Memory major pages are occurring at very high rate at {{`{{`}} $labels.instance {{`}}`}}, 500 major page faults per second for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + Please check that there is enough memory available at this instance. + summary: Memory major page faults are occurring at very high rate. + expr: | + rate(node_vmstat_pgmajfault{job="node-exporter"}[5m]) > 500 + for: 15m + labels: + severity: warning + - alert: NodeMemoryHighUtilization + annotations: + description: | + Memory is filling up at {{`{{`}} $labels.instance {{`}}`}}, has been above 90% for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}%. + summary: Host is running out of memory. + expr: | + 100 - (node_memory_MemAvailable_bytes{job="node-exporter"} / node_memory_MemTotal_bytes{job="node-exporter"} * 100) > 90 + for: 15m + labels: + severity: warning + - alert: NodeDiskIOSaturation + annotations: + description: | + Disk IO queue (aqu-sq) is high on {{`{{`}} $labels.device {{`}}`}} at {{`{{`}} $labels.instance {{`}}`}}, has been above 10 for the last 15 minutes, is currently at {{`{{`}} printf "%.2f" $value {{`}}`}}. + This symptom might indicate disk saturation. + summary: Disk IO queue is high. + expr: | + rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device!=""}[5m]) > 10 + for: 30m + labels: + severity: warning + - alert: NodeSystemdServiceFailed + annotations: + description: Systemd service {{`{{`}} $labels.name {{`}}`}} has entered failed state at {{`{{`}} $labels.instance {{`}}`}} + summary: Systemd service has entered failed state. + expr: | + node_systemd_unit_state{job="node-exporter", state="failed"} == 1 + for: 5m + labels: + severity: warning + - alert: NodeBondingDegraded + annotations: + description: Bonding interface {{`{{`}} $labels.master {{`}}`}} on {{`{{`}} $labels.instance {{`}}`}} is in degraded state due to one or more slave failures. + summary: Bonding interface is degraded + expr: | + (node_bonding_slaves - node_bonding_active) != 0 + for: 5m + labels: + severity: warning