Extend support for Prometheus exporters (#200)

* Make the serviceMonitor optional * Remove prometheusRule that is unsed from values * Optional serviceMonitor for PuppetDB Exporter * Make serviceMonitor optional * Update CHANGELOG and bump Chart version * Fix PuppetDB-Exporter container template vlaues * Add JMX exporter to compilers * Update tests * Enable serviceMonitor in serviceMonitor tests * Add more test coverage * add global values * Fix compiler labels in jmx templates * Add jmx to masters deployment and compiler statefulset * Move volume to correct location for pserver pods * Add annotations to PuppetDB and Masters * Update changelog * Add default puppetdb scrape interval * Add podAnnotations * Fix labels in serviceMonitor * Add more tests and cover everything with snapshots * Bump version and update changelog * Update snapshots to represent correct version in branch
puppetlabs · Dec 11, 2023 · 649b2ed · 649b2ed
1 parent cc723f6
commit 649b2ed
Show file tree

Hide file tree

Showing 40 changed files with 1,477 additions and 86 deletions.
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -1,4 +1,4 @@
-name: Lint and publish puppetserver helm chart
+name: CI / Release
 
 on:
   pull_request:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,13 @@ numbering uses [semantic versioning](http://semver.org).
 
 NOTE: The change log until version `v0.2.4` is auto-generated.
 
+## [v9.0.0](https://github.com/puppetlabs/puppetserver-helm-chart/tree/v9.0.0) (2023-12-08)
+- Breaking: Update values structure for 'metrics' to allow for multiple exporters
+- Feat: Make serviceMonitor optional for all Prometheus exporters
+- Feat: Add jmx exporter for puppetserver master(s)/compiler(s)
+- Feat: Allow specifying annotations for puppetserver master/compiler and puppetdb pods
+- Maint: Add snapshots to all unittests and over masters/compilers with them
+
 ## [v8.2.1](https://github.com/puppetlabs/puppetserver-helm-chart/tree/v8.2.0) (2023-11-29)
 - feat: GitHub Actions workflow: add lint, install and unittest for all PRs
 - feat: GitHub Actions workflow: simplify workflow_dispatch to aid releases from forks

diff --git a/Chart.yaml b/Chart.yaml
@@ -1,6 +1,6 @@
 apiVersion: v2
 name: puppetserver
-version: 8.2.1
+version: 9.0.0
 appVersion: 7.13.0
 description: Puppet automates the delivery and operation of software.
 keywords: ["puppet", "puppetserver", "automation", "iac", "infrastructure", "cm", "ci", "cd"]

diff --git a/templates/jmx-configmap.yaml b/templates/jmx-configmap.yaml
@@ -0,0 +1,13 @@
+{{- if .Values.metrics.prometheus.jmx.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "puppetserver.fullname" . }}-jmx-config
+  labels:
+    {{- include "puppetserver.puppetserver.labels" . | nindent 4 }}
+    {{- include "puppetserver.puppetserver-compilers.labels" . | nindent 4 }}
+data:
+  jmx_exporter.yaml: |-
+    {{ .Values.metrics.prometheus.jmx.config | nindent 4 }}
+{{- end }}
+
diff --git a/templates/jmx-servicemonitor.yaml b/templates/jmx-servicemonitor.yaml
@@ -0,0 +1,55 @@
+{{- if and .Values.metrics.prometheus.jmx.enabled .Values.metrics.prometheus.jmx.serviceMonitor.enabled  }}
+  {{- if (not (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1")) }}
+    {{- if (not (.Values.metrics.prometheus.disableAPICheck)) }}
+      {{- fail "ERROR: You have to deploy monitoring.coreos.com/v1 first" }}
+    {{- end }}
+  {{- end }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ template "puppetserver.fullname" . }}-jmx
+  namespace: {{ default .Release.Namespace .Values.metrics.prometheus.jmx.serviceMonitor.namespace | quote }}
+  labels:
+    {{- include "puppetserver.puppetserver.labels" . | nindent 4 }}
+    {{- with .Values.metrics.prometheus.jmx.serviceMonitor.additionalLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+    release: kube-prometheus-stack
+spec:
+  jobLabel: {{ .Values.metrics.prometheus.jmx.serviceMonitor.jobLabel | default "{{.Release.Name }}-jmx" }}
+  endpoints:
+    - port: metrics
+      path: /metrics
+      {{- with .Values.metrics.prometheus.jmx.serviceMonitor.honorLabels }}
+      honorLabels: {{ . }}
+      {{- end }}
+      {{- with .Values.metrics.prometheus.jmx.serviceMonitor.honorTimestamps }}
+      honorTimestamps: {{ . }}
+      {{- end }}
+      {{- with .Values.metrics.prometheus.jmx.serviceMonitor.enableHttp2 }}
+      enableHttp2: {{ . }}
+      {{- end }}
+      {{- with .Values.metrics.prometheus.jmx.serviceMonitor.followRedirects }}
+      followRedirects: {{ . }}
+      {{- end }}
+      {{- with .Values.metrics.prometheus.jmx.serviceMonitor.interval }}
+      interval: {{ . }}
+      {{- end }}
+      {{- with .Values.metrics.prometheus.jmx.serviceMonitor.scrapeTimeout }}
+      scrapeTimeout: {{ . }}
+      {{- end }}
+      {{- if .Values.metrics.prometheus.jmx.serviceMonitor.metricRelabelings }}
+      metricRelabelings:
+      {{ tpl (toYaml .Values.metrics.prometheus.jmx.serviceMonitor.metricRelabelings | indent 6) . }}
+      {{- end }}
+      {{- if .Values.metrics.prometheus.jmx.serviceMonitor.relabelings }}
+      relabelings:
+      {{ toYaml .Values.metrics.prometheus.jmx.serviceMonitor.relabelings | indent 6 }}
+      {{- end }}
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace | quote }}
+  selector:
+    matchLabels:
+      {{- include "puppetserver.puppetserver.matchLabels" . | nindent 6 }}
+{{- end }}
diff --git a/templates/puppetdb-deployment.yaml b/templates/puppetdb-deployment.yaml
@@ -8,6 +8,10 @@ metadata:
     {{- with .Values.puppetdb.extraLabels -}}
     {{ toYaml . | nindent 4 }}
     {{- end }}
+  {{- if .Values.puppetdb.annotations }}
+  annotations:
+    {{- toYaml .Values.puppetdb.annotations | nindent 4 }}
+  {{- end }}
 spec:
   selector:
     matchLabels:
@@ -32,6 +36,9 @@ spec:
         {{- if .Values.podAnnotations }}
         {{- toYaml .Values.podAnnotations | nindent 8 }}
         {{- end }}
+        {{- if .Values.puppetdb.podAnnotations }}
+        {{- toYaml .Values.puppetdb.podAnnotations | nindent 8 }}
+        {{- end }}
     spec:
       hostname: puppetdb
       {{- if .Values.puppetdb.serviceAccount.enabled }}
@@ -265,12 +272,12 @@ spec:
             successThreshold: 1
             timeoutSeconds: 5
         {{- end }}
-        {{- if .Values.metrics.prometheus.enabled  }}
+        {{- if .Values.metrics.prometheus.puppetdb.enabled  }}
         - name: puppetdb-exporter
-          image: "{{ tpl .Values.metrics.prometheus.image . }}:{{ tpl .Values.metrics.prometheus.tag . }}"
-          imagePullPolicy: {{ tpl .Values.metrics.prometheus.imagePullPolicy . }}
+          image: "{{ tpl .Values.metrics.prometheus.puppetdb.image . }}:{{ tpl .Values.metrics.prometheus.puppetdb.tag . }}"
+          imagePullPolicy: {{ tpl .Values.metrics.prometheus.puppetdb.imagePullPolicy . }}
           resources:
-            {{- toYaml .Values.metrics.prometheus.resources | nindent 12 }}
+            {{- toYaml .Values.metrics.prometheus.puppetdb.resources | nindent 12 }}
           env:
             - name: PUPPETDB_URL
               value: "https://{{ if .Values.singleCA.enabled}}{{.Values.singleCA.puppetdb.overrideHostname}}{{ else }}{{ ( include "puppetdb.fullname" . ) }}{{ end }}:8081/pdb/query"
@@ -281,14 +288,14 @@ spec:
             - name: PUPPETDB_CA_FILE
               value: "/opt/puppetlabs/server/data/puppetdb/certs/certs/ca.pem"
             - name: PUPPETDB_SCRAPE_INTERVAL
-              value: "{{ .Values.metrics.prometheus.interval }}"
+              value: "{{ .Values.metrics.prometheus.puppetdb.interval }}"
             - name: PUPPETDB_LISTEN_ADDRESS
-              value: "0.0.0.0:{{ .Values.metrics.prometheus.port }}"
+              value: "0.0.0.0:{{ .Values.metrics.prometheus.puppetdb.port }}"
             {{- range $key, $value := .Values.global.extraEnv }}
             - name: {{ $key }}
               value: "{{ $value }}"
             {{- end }}
-            {{- range $key, $value := .Values.metrics.prometheus.extraEnv }}
+            {{- range $key, $value := .Values.metrics.prometheus.puppetdb.extraEnv }}
             - name: {{ $key }}
               value: "{{ $value }}"
             {{- end }}
@@ -297,13 +304,13 @@ spec:
             - secretRef:
                 name: {{ .Values.global.extraEnvSecret }}
           {{- end }}
-          {{- if .Values.metrics.prometheus.extraEnvSecret }}
+          {{- if .Values.metrics.prometheus.puppetdb.extraEnvSecret }}
             - secretRef:
-                name: {{ .Values.metrics.prometheus.extraEnvSecret }}
+                name: {{ .Values.metrics.prometheus.puppetdb.extraEnvSecret }}
           {{- end }}
           ports:
             - name: metrics
-              containerPort: {{ .Values.metrics.prometheus.port }}
+              containerPort: {{ .Values.metrics.prometheus.puppetdb.port }}
           securityContext:
             runAsUser: 999
             runAsGroup: 999

diff --git a/templates/puppetdb-servicemonitor.yaml b/templates/puppetdb-servicemonitor.yaml
@@ -1,4 +1,4 @@
-{{- if and .Values.puppetdb.enabled .Values.metrics.prometheus.enabled }}
+{{- if and .Values.puppetdb.enabled .Values.metrics.prometheus.puppetdb.enabled .Values.metrics.prometheus.puppetdb.serviceMonitor.enabled  }}
   {{- if (not (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1")) }}
     {{- if (not (.Values.metrics.prometheus.disableAPICheck)) }}
       {{- fail "ERROR: You have to deploy monitoring.coreos.com/v1 first" }}
@@ -8,48 +8,48 @@ apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
   name: {{ template "puppetdb.fullname" . }}
-  namespace: {{ default .Release.Namespace .Values.metrics.prometheus.namespace | quote }}
+  namespace: {{ default .Release.Namespace .Values.metrics.prometheus.puppetdb.serviceMonitor.namespace | quote }}
   labels:
     {{- include "puppetserver.puppetdb.labels" . | nindent 4 }}
-    {{- with .Values.metrics.prometheus.additionalLabels }}
+    {{- with .Values.metrics.prometheus.puppetdb.serviceMonitor.additionalLabels }}
     {{- toYaml . | nindent 4 }}
     {{- end }}
     release: kube-prometheus-stack
 spec:
-  jobLabel: {{ .Values.metrics.prometheus.jobLabel | default "{{.Release.Name }}-puppetdb" }}
+  jobLabel: {{ .Values.metrics.prometheus.puppetdb.serviceMonitor.jobLabel | default "{{.Release.Name }}-puppetdb" }}
   endpoints:
     - port: metrics
       path: /metrics
-      {{- with .Values.metrics.prometheus.honorLabels }}
+      {{- with .Values.metrics.prometheus.puppetdb.serviceMonitor.honorLabels }}
       honorLabels: {{ . }}
       {{- end }}
-      {{- with .Values.metrics.prometheus.honorTimestamps }}
+      {{- with .Values.metrics.prometheus.puppetdb.serviceMonitor.honorTimestamps }}
       honorTimestamps: {{ . }}
       {{- end }}
-      {{- with .Values.metrics.prometheus.enableHttp2 }}
+      {{- with .Values.metrics.prometheus.puppetdb.serviceMonitor.enableHttp2 }}
       enableHttp2: {{ . }}
       {{- end }}
-      {{- with .Values.metrics.prometheus.followRedirects }}
+      {{- with .Values.metrics.prometheus.puppetdb.serviceMonitor.followRedirects }}
       followRedirects: {{ . }}
       {{- end }}
-      {{- with .Values.metrics.prometheus.interval }}
+      {{- with .Values.metrics.prometheus.puppetdb.serviceMonitor.interval }}
       interval: {{ . }}
       {{- end }}
-      {{- with .Values.metrics.prometheus.scrapeTimeout }}
+      {{- with .Values.metrics.prometheus.puppetdb.serviceMonitor.scrapeTimeout }}
       scrapeTimeout: {{ . }}
       {{- end }}
-      {{- if .Values.metrics.prometheus.metricRelabelings }}
+      {{- if .Values.metrics.prometheus.puppetdb.serviceMonitor.metricRelabelings }}
       metricRelabelings:
-      {{ tpl (toYaml .Values.metrics.prometheus.metricRelabelings | indent 6) . }}
+      {{ tpl (toYaml .Values.metrics.prometheus.puppetdb.serviceMonitor.metricRelabelings | indent 6) . }}
       {{- end }}
-      {{- if .Values.metrics.prometheus.relabelings }}
+      {{- if .Values.metrics.prometheus.puppetdb.serviceMonitor.relabelings }}
       relabelings:
-      {{ toYaml .Values.metrics.prometheus.relabelings | indent 6 }}
+      {{ toYaml .Values.metrics.prometheus.puppetdb.serviceMonitor.relabelings | indent 6 }}
       {{- end }}
   namespaceSelector:
     matchNames:
       - {{ .Release.Namespace | quote }}
   selector:
     matchLabels:
       {{- include "puppetserver.puppetdb.matchLabels" . | nindent 6 }}
-{{- end }}
+{{- end }}
diff --git a/templates/puppetserver-deployment-compilers.yaml b/templates/puppetserver-deployment-compilers.yaml
@@ -47,6 +47,9 @@ spec:
         {{- if .Values.podAnnotations }}
         {{- toYaml .Values.podAnnotations | nindent 8 }}
         {{- end }}
+        {{- if .Values.puppetserver.compilers.podAnnotations }}
+        {{- toYaml .Values.puppetserver.compilers.podAnnotations | nindent 8 }}
+        {{- end }}
     spec:
       hostname: puppet
       {{- if .Values.puppetserver.serviceAccount.enabled }}
@@ -493,9 +496,51 @@ spec:
             successThreshold: 1
             timeoutSeconds: 5
         {{- end }}
+        {{- if .Values.metrics.prometheus.jmx.enabled  }}
+        - name: jmx-exporter
+          image: "{{ tpl .Values.metrics.prometheus.jmx.image . }}:{{ tpl .Values.metrics.prometheus.jmx.tag . }}"
+          imagePullPolicy: {{ tpl .Values.metrics.prometheus.jmx.imagePullPolicy . }}
+          resources:
+            {{- toYaml .Values.metrics.prometheus.jmx.resources | nindent 12 }}
+          args: ["{{ .Values.metrics.prometheus.jmx.port }}", "/jmx_exporter.yaml"]
+          env:
+            {{- range $key, $value := .Values.global.extraEnv }}
+            - name: {{ $key }}
+              value: "{{ $value }}"
+            {{- end }}
+            {{- range $key, $value := .Values.metrics.prometheus.jmx.extraEnv }}
+            - name: {{ $key }}
+              value: "{{ $value }}"
+            {{- end }}
+          envFrom:
+          {{- if .Values.global.extraEnvSecret }}
+            - secretRef:
+                name: {{ .Values.global.extraEnvSecret }}
+          {{- end }}
+          {{- if .Values.metrics.prometheus.jmx.extraEnvSecret }}
+            - secretRef:
+                name: {{ .Values.metrics.prometheus.jmx.extraEnvSecret }}
+          {{- end }}
+          ports:
+            - name: metrics
+              containerPort: {{ .Values.metrics.prometheus.jmx.port }}
+          volumeMounts:
+            - name: jmx-config
+              mountPath: /jmx_exporter.yaml
+              subPath: jmx_exporter.yaml
+          securityContext:
+            runAsUser: 999
+            runAsGroup: 999
+            runAsNonRoot: true
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - all
+        {{- end }}
         {{- with .Values.puppetserver.compilers.extraContainers }}
           {{- toYaml . | nindent 8 }}
         {{- end }}
+
       {{- if .Values.singleCA.puppetdb.overrideHostname }}
       {{- $service := lookup "v1" "Service" .Release.Namespace  (include "puppetdb.fullname" . ) }}
       hostAliases:
@@ -539,6 +584,11 @@ spec:
             name: {{ template "puppetserver.fullname" . }}-customentrypoints
             defaultMode: 0777
         {{- end }}
+        {{- if .Values.metrics.prometheus.jmx.enabled  }}
+        - name: jmx-config
+          configMap:
+            name: {{ template "puppetserver.fullname" . }}-jmx-config
+        {{- end }}
         - name: manifests-volume
           configMap:
             name: {{ template "puppetserver.fullname" . }}-manifests-config

diff --git a/templates/puppetserver-deployment-masters.yaml b/templates/puppetserver-deployment-masters.yaml
@@ -7,6 +7,10 @@ metadata:
     {{- with .Values.puppetserver.masters.extraLabels -}}
     {{ toYaml . | nindent 4 }}
     {{- end }}
+  {{- if .Values.puppetserver.masters.annotations }}
+  annotations:
+    {{- toYaml .Values.puppetserver.masters.annotations | nindent 4 }}
+  {{- end }}
 spec:
   {{- if .Values.puppetserver.masters.multiMasters.enabled }}
   {{- if not (.Values.puppetserver.masters.multiMasters.autoScaling.enabled) }}
@@ -44,6 +48,9 @@ spec:
         {{- if .Values.podAnnotations }}
         {{- toYaml .Values.podAnnotations | nindent 8 }}
         {{- end }}
+        {{- if .Values.puppetserver.masters.podAnnotations }}
+        {{- toYaml .Values.puppetserver.masters.podAnnotations | nindent 8 }}
+        {{- end }}
     spec:
       hostname: puppet
       {{- if .Values.puppetserver.serviceAccount.enabled }}
@@ -520,6 +527,47 @@ spec:
             successThreshold: 1
             timeoutSeconds: 5
         {{- end }}
+        {{- if .Values.metrics.prometheus.jmx.enabled  }}
+        - name: jmx-exporter
+          image: "{{ tpl .Values.metrics.prometheus.jmx.image . }}:{{ tpl .Values.metrics.prometheus.jmx.tag . }}"
+          imagePullPolicy: {{ tpl .Values.metrics.prometheus.jmx.imagePullPolicy . }}
+          resources:
+            {{- toYaml .Values.metrics.prometheus.jmx.resources | nindent 12 }}
+          args: ["{{ .Values.metrics.prometheus.jmx.port }}", "/jmx_exporter.yaml"]
+          env:
+            {{- range $key, $value := .Values.global.extraEnv }}
+            - name: {{ $key }}
+              value: "{{ $value }}"
+            {{- end }}
+            {{- range $key, $value := .Values.metrics.prometheus.jmx.extraEnv }}
+            - name: {{ $key }}
+              value: "{{ $value }}"
+            {{- end }}
+          envFrom:
+          {{- if .Values.global.extraEnvSecret }}
+            - secretRef:
+                name: {{ .Values.global.extraEnvSecret }}
+          {{- end }}
+          {{- if .Values.metrics.prometheus.jmx.extraEnvSecret }}
+            - secretRef:
+                name: {{ .Values.metrics.prometheus.jmx.extraEnvSecret }}
+          {{- end }}
+          ports:
+            - name: metrics
+              containerPort: {{ .Values.metrics.prometheus.jmx.port }}
+          volumeMounts:
+            - name: jmx-config
+              mountPath: /jmx_exporter.yaml
+              subPath: jmx_exporter.yaml
+          securityContext:
+            runAsUser: 999
+            runAsGroup: 999
+            runAsNonRoot: true
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - all
+        {{- end }}
         {{- with .Values.puppetserver.masters.extraContainers }}
           {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -576,6 +624,11 @@ spec:
             name: {{ template "puppetserver.fullname" . }}-customentrypoints
             defaultMode: 0777
         {{- end }}
+        {{- if .Values.metrics.prometheus.jmx.enabled  }}
+        - name: jmx-config
+          configMap:
+            name: {{ template "puppetserver.fullname" . }}-jmx-config
+        {{- end }}
         - name: manifests-volume
           configMap:
             name: {{ template "puppetserver.fullname" . }}-manifests-config