Add Prometheus alerting functionality

.github/workflows/e2e-test.yml

@@ -97,4 +97,15 @@ jobs:
 
     - name: Exec OpenIM System uninstall
       run: |
-        sudo ./scripts/install/install.sh -u
+        sudo ./scripts/install/install.sh -u
+
+    - name: gobenchdata publish
+      uses: bobheadxi/gobenchdata@v1
+      with:
+        PRUNE_COUNT: 30
+        GO_TEST_FLAGS: -cpu 1,2
+        PUBLISH: true
+        PUBLISH_BRANCH: gh-pages
+      env:
+        GITHUB_TOKEN: ${{ secrets.BOT_GITHUB_TOKEN }}
+      continue-on-error: true

.github/workflows/link-pr.yml

@@ -41,7 +41,7 @@ jobs:
             # ./*.md all markdown files in the root directory
           args: --verbose  -E -i --no-progress --exclude-path './CHANGELOG' './**/*.md'
         env:
-          GITHUB_TOKEN: ${{secrets.GH_PAT}}
+          GITHUB_TOKEN: ${{secrets.BOT_GITHUB_TOKEN}}
 
       - name: Create Issue From File
         if: env.lychee_exit_code != 0

README.md

@@ -231,7 +231,7 @@ Before you start, please make sure your changes are in demand. The best for that
 - [OpenIM Makefile Utilities](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/util-makefile.md)
 - [OpenIM Script Utilities](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/util-scripts.md)
 - [OpenIM Versioning](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/version.md)
-
+- [Manage backend and monitor deployment](https://github.com/openimsdk/open-im-server/tree/main/docs/contrib/prometheus-grafana.md)
 
 ## :busts_in_silhouette: Community
 

config/alertmanager.yml

@@ -0,0 +1,32 @@
+###################### AlertManager Configuration ######################
+# AlertManager configuration using environment variables
+#
+# Resolve timeout
+# SMTP configuration for sending alerts
+# Templates for email notifications
+# Routing configurations for alerts
+# Receiver configurations
+global:
+  resolve_timeout: 5m
+  smtp_from: alert@openim.io
+  smtp_smarthost: smtp.163.com:465
+  smtp_auth_username: alert@openim.io
+  smtp_auth_password: YOURAUTHPASSWORD
+  smtp_require_tls: false
+  smtp_hello: xxx监控告警
+
+templates:
+  - /etc/alertmanager/email.tmpl
+
+route:
+  group_wait: 5s
+  group_interval: 5s
+  repeat_interval: 5m
+  receiver: email
+receivers:
+  - name: email
+    email_configs:
+      - to: {EMAIL_TO:-'alert@example.com'}
+        html: '{{ template "email.to.html" . }}'
+        headers: { Subject: "[OPENIM-SERVER]Alarm" }
+        send_resolved: true

config/email.tmpl

@@ -0,0 +1,16 @@
+{{ define "email.to.html" }}
+{{ range .Alerts }}
+<!-- Begin of OpenIM Alert -->
+<div style="border:1px solid #ccc; padding:10px; margin-bottom:10px;">
+    <h3>OpenIM Alert</h3>
+    <p><strong>Alert Program:</strong> Prometheus Alert</p>
+    <p><strong>Severity Level:</strong> {{ .Labels.severity }}</p>
+    <p><strong>Alert Type:</strong> {{ .Labels.alertname }}</p>
+    <p><strong>Affected Host:</strong> {{ .Labels.instance }}</p>
+    <p><strong>Affected Service:</strong> {{ .Labels.job }}</p>
+    <p><strong>Alert Subject:</strong> {{ .Annotations.summary }}</p>
+    <p><strong>Trigger Time:</strong> {{ .StartsAt.Format "2006-01-02 15:04:05" }}</p>
+</div>
+<!-- End of OpenIM Alert -->
+{{ end }}
+{{ end }}

config/instance-down-rules.yml

@@ -0,0 +1,11 @@
+groups:
+  - name: instance_down
+    rules:
+      - alert: InstanceDown
+        expr: up == 0
+        for: 1m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Instance {{ $labels.instance }} down"
+          description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 1 minutes."

config/prometheus.yml

@@ -6,13 +6,13 @@ global:
 
 # Alertmanager configuration
 alerting:
-#alertmanagers:
-#  - static_configs:
-#      - targets: ['172.29.166.17:9093'] #alertmanager地址
+  alertmanagers:
+    - static_configs:
+        - targets: ['172.28.0.1:19093']
 
 # Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
 rule_files:
-#  - "node_down.yml"
+  - "instance-down-rules.yml"
 # - "first_rules.yml"
 # - "second_rules.yml"
 

deployments/templates/alertmanager.yml

@@ -0,0 +1,32 @@
+###################### AlertManager Configuration ######################
+# AlertManager configuration using environment variables
+#
+# Resolve timeout
+# SMTP configuration for sending alerts
+# Templates for email notifications
+# Routing configurations for alerts
+# Receiver configurations
+global:
+  resolve_timeout: ${ALERTMANAGER_RESOLVE_TIMEOUT}
+  smtp_from: ${ALERTMANAGER_SMTP_FROM}
+  smtp_smarthost: ${ALERTMANAGER_SMTP_SMARTHOST}
+  smtp_auth_username: ${ALERTMANAGER_SMTP_AUTH_USERNAME}
+  smtp_auth_password: ${ALERTMANAGER_SMTP_AUTH_PASSWORD}
+  smtp_require_tls: ${ALERTMANAGER_SMTP_REQUIRE_TLS}
+  smtp_hello: ${ALERTMANAGER_SMTP_HELLO}
+
+templates:
+  - /etc/alertmanager/email.tmpl
+
+route:
+  group_wait: 5s
+  group_interval: 5s
+  repeat_interval: 5m
+  receiver: email
+receivers:
+  - name: email
+    email_configs:
+      - to: ${ALERTMANAGER_EMAIL_TO}
+        html: '{{ template "email.to.html" . }}'
+        headers: { Subject: "[OPENIM-SERVER]Alarm" }
+        send_resolved: true

deployments/templates/env_template.yaml

@@ -94,7 +94,7 @@ OPENIM_CHAT_NETWORK_ADDRESS=${OPENIM_CHAT_NETWORK_ADDRESS}
 # Address or hostname for the Prometheus network.
 # Default: PROMETHEUS_NETWORK_ADDRESS=172.28.0.11
 PROMETHEUS_NETWORK_ADDRESS=${PROMETHEUS_NETWORK_ADDRESS}
-
+  
 # Address or hostname for the Grafana network.
 # Default: GRAFANA_NETWORK_ADDRESS=172.28.0.12
 GRAFANA_NETWORK_ADDRESS=${GRAFANA_NETWORK_ADDRESS}
@@ -106,7 +106,10 @@ NODE_EXPORTER_NETWORK_ADDRESS=${NODE_EXPORTER_NETWORK_ADDRESS}
 # Address or hostname for the OpenIM admin network.
 # Default: OPENIM_ADMIN_NETWORK_ADDRESS=172.28.0.14
 OPENIM_ADMIN_FRONT_NETWORK_ADDRESS=${OPENIM_ADMIN_FRONT_NETWORK_ADDRESS}
-
+
+# Address or hostname for the alertmanager network.
+# Default: ALERT_MANAGER_NETWORK_ADDRESS=172.28.0.15
+ALERT_MANAGER_NETWORK_ADDRESS=${ALERT_MANAGER_NETWORK_ADDRESS}
 # ===============================================
 # = Component Extension Configuration =
 # ===============================================
@@ -305,4 +308,8 @@ GRAFANA_PORT=${GRAFANA_PORT}
 
 # Port for the admin front.
 # Default: OPENIM_ADMIN_FRONT_PORT=11002
-OPENIM_ADMIN_FRONT_PORT=${OPENIM_ADMIN_FRONT_PORT}
+OPENIM_ADMIN_FRONT_PORT=${OPENIM_ADMIN_FRONT_PORT}
+
+# Port for the alertmanager.
+# Default: ALERT_MANAGER_PORT=19093
+ALERT_MANAGER_PORT=${ALERT_MANAGER_PORT}

deployments/templates/prometheus.yml

@@ -6,13 +6,13 @@ global:
 
 # Alertmanager configuration
 alerting:
-#alertmanagers:
-#  - static_configs:
-#      - targets: ['172.29.166.17:9093'] #alertmanager地址
+  alertmanagers:
+    - static_configs:
+        - targets: ['${ALERT_MANAGER_ADDRESS}:${ALERT_MANAGER_PORT}']
 
 # Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
 rule_files:
-#  - "node_down.yml"
+  - "instance-down-rules.yml"
 # - "first_rules.yml"
 # - "second_rules.yml"
 

docker-compose.yml

@@ -162,12 +162,27 @@ services:
     restart: always
     volumes:
       - ./config/prometheus.yml:/etc/prometheus/prometheus.yml
+      - ./config/instance-down-rules.yml:/etc/prometheus/instance-down-rules.yml
     ports:
       - "${PROMETHEUS_PORT}:9090"
     networks:
       server:
         ipv4_address: ${PROMETHEUS_NETWORK_ADDRESS}
 
+  alertmanager:
+    image: prom/alertmanager
+    container_name: alertmanager
+    hostname: alertmanager
+    restart: always
+    volumes:
+      - ./config/alertmanager.yml:/etc/alertmanager/alertmanager.yml
+      - ./config/email.tmpl:/etc/alertmanager/email.tmpl
+    ports:
+      - "${ALERT_MANAGER_PORT}:9093"
+    networks:
+      server:
+        ipv4_address: ${ALERT_MANAGER_NETWORK_ADDRESS}
+
   grafana:
     image: grafana/grafana
     container_name: grafana