feat: Add possibility to collect MQTT broker metrics

helm/flowforge/README.md

@@ -87,6 +87,7 @@ To use STMP to send email
   - `forge.broker.url` URL to access the broker from inside the cluster (default `mqtt://flowforge-broker.[namespace]:1883`)
   - `forge.broker.public_url` URL to access the broker from outside the cluster (default `ws://mqtt.[forge.domain]`, uses `wss://` if `forge.https` is `true`)
   - `forge.broker.hostname` the custom Fully Qualified Domain Name (FQDN) where the broker will be hosted (default `mqtt.[forge.domain]`)
+  - `forge.broker.createMetricsUser` defines if a dedicated MQTT user with broker metrics collection permissions should be created (default `true`)
   - `forge.broker.affinity` allows to configure [affinity or anti-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity) for the broker pod
   - `forge.broker.resources` allows to configure [resources](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) for the broker container
   - `forge.broker.podSecurityContext` allows to configure [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) for the broker pod
@@ -103,6 +104,8 @@ To use STMP to send email
   - `{{ instanceHost }}` replaced by the hostname of the instance
   - `{{ serviceName }}` replaced by the service name of the instance
 
+`forge.broker.createMetricsUser` parameter controlls if a dedicated MQTT user with broker metrics collection permissions should be created. This user can by used by the tools like [Mosquitto Exporter](https://github.com/sapcc/mosquitto-exporter) to expose broker's metrics for Prometheus scrapper. 
+
 ### Telemetry
 
 Enables FlowForge Telemetry

helm/flowforge/templates/NOTES.txt

@@ -1,3 +1,10 @@
 Thank you for installing {{ .Chart.Name }} v{{ .Chart.AppVersion }}
 
-You can complete the setup wizard at {{ if .Values.forge.entryPoint }}http{{- if .Values.forge.https -}}s{{- end -}}://{{ .Values.forge.entryPoint }}{{- else }}http{{- if .Values.forge.https -}}s{{- end -}}://forge.{{ .Values.forge.domain }}{{- end }}
+You can complete the setup wizard at {{ if .Values.forge.entryPoint }}http{{- if .Values.forge.https -}}s{{- end -}}://{{ .Values.forge.entryPoint }}{{- else }}http{{- if .Values.forge.https -}}s{{- end -}}://forge.{{ .Values.forge.domain }}{{- end }}
+
+{{- if .Values.forge.broker.createMetricsUser }}
+You configured the installation to create a dedicated MQTT user for collecting broker's metrics. 
+You can access the generated values by executing:
+kubectl -n {{ .Release.Namespace }} get secret flowfuse-broker-secrets -o jsonpath='{.data.metrics_password}' | base64 -d
+kubectl -n {{ .Release.Namespace }} get secret flowfuse-broker-secrets -o jsonpath='{.data.metrics_user}' | base64 -d
+{{- end }}

helm/flowforge/templates/broker-config.yaml

@@ -0,0 +1,95 @@
+{{- if .Values.forge.broker.enabled -}}
+{{- $metricsUser := "metrics_reader" }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: flowforge-broker-config
+  labels:
+    {{- include "forge.labels" . | nindent 4 }}
+data:
+  mosquitto.conf: |
+    per_listener_settings false
+    allow_anonymous false
+
+    listener 1883 0.0.0.0
+    listener 1884 0.0.0.0
+    protocol websockets
+    http_dir /http
+
+    auth_plugin /mosquitto/go-auth.so
+    auth_opt_hasher bcrypt
+    auth_opt_cache true
+    auth_opt_auth_cache_seconds 120
+    auth_opt_acl_cache_seconds 300
+    auth_opt_auth_jitter_second 3
+    auth_opt_acl_jitter_seconds 5
+    auth_opt_check_prefix true
+    {{- if .Values.forge.broker.createMetricsUser }}
+    auth_opt_backends files, http
+    auth_opt_prefixes metrics, forge
+
+    auth_opt_files_password_path /etc/mosquitto/password_file
+    auth_opt_files_acl_path /etc/mosquitto/acl_file
+    {{- else }}
+    auth_opt_backends http
+    auth_opt_prefixes forge
+    {{- end }}
+
+    auth_opt_http_host forge.{{ .Release.Namespace }}
+    auth_opt_http_port 80
+    auth_opt_http_getuser_uri /api/comms/auth/client
+    auth_opt_http_aclcheck_uri /api/comms/auth/acl
+
+  {{- if .Values.forge.broker.createMetricsUser }}
+  mqtt_acl_file: |
+    user {{ $metricsUser }}
+    topic read $SYS/#
+  {{- end }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: flowforge-broker-ping
+  labels:
+    {{- include "forge.labels" . | nindent 4 }}
+data:
+  ping.html: |
+    <html>
+      <head>
+        <title>Mosquitto Liveness Check</title>
+        <body>
+          <h1>HelloWorld</h1>
+        </body>
+      </head>
+    </html>
+{{- if .Values.forge.broker.createMetricsUser }}
+{{- $secretName := "flowfuse-broker-secrets" }}
+{{- $existingSecret := (lookup "v1" "Secret" .Release.Namespace $secretName) | default dict }}
+{{- $metricsPassword := "" }}
+{{- $metricsPassword := "" }}
+{{- $mqttPasswordFile := "" }}
+{{- if and $existingSecret.data (hasKey $existingSecret.data "metrics_password") }}
+{{- $metricsPassword = $existingSecret.data.metrics_password | b64dec }}
+{{- else }}
+{{- $metricsPassword = randAlphaNum 32 }}
+{{- end }}
+{{- if and $existingSecret.data (hasKey $existingSecret.data "mqtt_password_file") }}
+{{- $mqttPasswordFile = $existingSecret.data.mqtt_password_file | b64dec }}
+{{- else }}
+{{- $mqttPasswordFile = htpasswd $metricsUser $metricsPassword }}
+{{- end }}
+---
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: {{ $secretName }}
+  labels:
+    {{- include "forge.labels" . | nindent 4 }}
+data:
+  metrics_user: {{ $metricsUser | b64enc | quote }}
+  metrics_password: {{ $metricsPassword | b64enc | quote }}
+  mqtt_password_file: {{ $mqttPasswordFile | b64enc | quote }}
+
+{{- end }}
+{{- end }}

helm/flowforge/templates/broker.yaml

@@ -1,51 +1,4 @@
 {{- if .Values.forge.broker.enabled -}}
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: flowforge-broker-config
-  labels:
-    {{- include "forge.labels" . | nindent 4 }}
-data:
-  mosquitto.conf: |
-    per_listener_settings false
-    allow_anonymous false
-
-    listener 1883 0.0.0.0
-    listener 1884 0.0.0.0
-    protocol websockets
-    http_dir /http
-
-    auth_plugin /mosquitto/go-auth.so
-    auth_opt_backends http
-    auth_opt_hasher bcrypt
-    auth_opt_cache true
-    auth_opt_auth_cache_seconds 120
-    auth_opt_acl_cache_seconds 300
-    auth_opt_auth_jitter_second 3
-    auth_opt_acl_jitter_seconds 5
-
-    auth_opt_http_host forge.{{ .Release.Namespace }}
-    auth_opt_http_port 80
-    auth_opt_http_getuser_uri /api/comms/auth/client
-    auth_opt_http_aclcheck_uri /api/comms/auth/acl
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: flowforge-broker-ping
-  labels:
-    {{- include "forge.labels" . | nindent 4 }}
-data:
-  ping.html: |
-    <html>
-      <head>
-        <title>Mosquitto Liveness Check</title>
-        <body>
-          <h1>HelloWorld</h1>
-        </body>
-      </head>
-    </html>
----
 apiVersion: apps/v1
 kind: Deployment
 metadata: 
@@ -67,6 +20,8 @@ spec:
   replicas: 1
   template:
     metadata:
+      annotations:
+        checksum/config: {{ include (print $.Template.BasePath "/broker-config.yaml") . | sha256sum }}
       labels:
         {{- include "forge.brokerSelectorLabels" . | nindent 8 }}
         {{- with .Values.forge.broker.podLabels }}
@@ -82,9 +37,19 @@ spec:
         imagePullPolicy: Always
         volumeMounts:
         - name: config
-          mountPath: /etc/mosquitto
+          mountPath: /etc/mosquitto/mosquitto.conf
+          subPath: mosquitto.conf
         - name: ping
           mountPath: /http
+        {{- if .Values.forge.broker.createMetricsUser }}
+        - name: config
+          mountPath: /etc/mosquitto/acl_file
+          subPath: mqtt_acl_file
+        - name: secrets
+          mountPath: /etc/mosquitto/password_file
+          subPath: mqtt_password_file
+          readOnly: true
+        {{- end }}
         ports:
         - containerPort: 1883
           name: mqtt-native
@@ -140,6 +105,11 @@ spec:
       - name: ping
         configMap:
           name: flowforge-broker-ping
+      {{- if .Values.forge.broker.createMetricsUser }}
+      - name: secrets
+        secret:
+          secretName: flowfuse-broker-secrets
+      {{- end }}
       {{- if .Values.forge.managementSelector }}
       nodeSelector:
         {{- range $key, $value := .Values.forge.managementSelector }}

helm/flowforge/values.schema.json

@@ -345,6 +345,9 @@
                                     "minProperties": 0
                                 }
                             }
+                        },
+                        "createMetricsUser": {
+                            "type": "boolean"
                         }
                     },
                     "required": [

helm/flowforge/values.yaml

@@ -19,6 +19,7 @@ forge:
   podLabels: {}
   broker:
     enabled: false
+    createMetricsUser: false
     podSecurityContext:
       runAsUser: 1000
       runAsGroup: 1000