From 827539b4fa1f09181b594d93f8fe5de3666fbe3c Mon Sep 17 00:00:00 2001 From: Maurice Faber Date: Thu, 16 Jun 2022 17:12:19 +0200 Subject: [PATCH] fix: servicemonitor labels (#822) also fixed loki regression by adding retention enabled flag, removed unnecessary monitor --- binzx/otomi | 2 +- .../grafana/templates/servicemonitor.yaml | 2 +- helmfile.d/helmfile-60.teams.yaml | 8 ++ values/loki/loki.gotmpl | 1 + values/prometheus-operator/pod-monitors.yaml | 18 +-- .../prometheus-operator.gotmpl | 26 +++- .../prometheus-operator/service-monitors.yaml | 128 ++++++++---------- 7 files changed, 104 insertions(+), 81 deletions(-) diff --git a/binzx/otomi b/binzx/otomi index 3c16faf89e..82bbb5b983 100755 --- a/binzx/otomi +++ b/binzx/otomi @@ -27,7 +27,7 @@ elif [ -z "$IN_DOCKER" ] && [ -n "$ENV_DIR" ]; then fi [[ "$ENV_DIR" == *"../"* ]] && echo "Don't provide an ENV_DIR that contains '../'!" && exit 1 # treat CI and chart as non interactive -if [ -z "$PS1" ]; then +if [[ $calling_args != 'bash '* ]] && [[ $calling_args != 'x '* ]] && { [ -z "$PS1" ] || [ -n "$CI" ]; }; then calling_args="$calling_args --non-interactive" fi if [[ $calling_args == 'x '* ]] && [[ $calling_args != 'x -- '* ]]; then diff --git a/charts/prometheus-operator/charts/grafana/templates/servicemonitor.yaml b/charts/prometheus-operator/charts/grafana/templates/servicemonitor.yaml index 4b6437e690..3658c075a4 100644 --- a/charts/prometheus-operator/charts/grafana/templates/servicemonitor.yaml +++ b/charts/prometheus-operator/charts/grafana/templates/servicemonitor.yaml @@ -38,5 +38,5 @@ spec: {{- include "grafana.selectorLabels" . | nindent 8 }} namespaceSelector: matchNames: - - {{ .Release.Namespace }} + - {{ template "grafana.namespace" . }} {{- end }} diff --git a/helmfile.d/helmfile-60.teams.yaml b/helmfile.d/helmfile-60.teams.yaml index 9d714617e1..55ccd894fd 100644 --- a/helmfile.d/helmfile-60.teams.yaml +++ b/helmfile.d/helmfile-60.teams.yaml @@ -81,10 +81,18 @@ releases: alertmanager: false enabled: false {{- end }} + commonLabels: + prometheus: team-{{ $teamId }} prometheus: namespaceOverride: team-{{ $teamId }} prometheusSpec: externalUrl: https://{{ $appsDomain }}/prometheus + podMonitorSelector: + matchLabels: + prometheus: team-{{ $teamId }} + serviceMonitorSelector: + matchLabels: + prometheus: team-{{ $teamId }} ruleNamespaceSelector: matchLabels: name: team-{{ $teamId }} diff --git a/values/loki/loki.gotmpl b/values/loki/loki.gotmpl index 4ffe559d14..bf94dd6d5c 100644 --- a/values/loki/loki.gotmpl +++ b/values/loki/loki.gotmpl @@ -44,6 +44,7 @@ config: {{ $l.storageType }}: {{- $l | get $l.storageType | toYaml | nindent 6 }} {{- end }} compactor: + retention_enabled: true shared_store: {{ $l | get "storageType" "filesystem" }} chunk_store_config: max_look_back_period: {{ $l | get "retention.period" "24h" }} diff --git a/values/prometheus-operator/pod-monitors.yaml b/values/prometheus-operator/pod-monitors.yaml index 74f6d05716..67ee61cb5c 100644 --- a/values/prometheus-operator/pod-monitors.yaml +++ b/values/prometheus-operator/pod-monitors.yaml @@ -1,8 +1,10 @@ -- name: istio-sidecars - namespace: istio-system - selector: - matchLabels: - security.istio.io/tlsMode: istio - podMetricsEndpoints: - - port: http-envoy-prom - path: /stats/prometheus +additionalPodMonitors: + - name: istio-sidecars + selector: + matchLabels: + security.istio.io/tlsMode: istio + namespaceSelector: + any: true + podMetricsEndpoints: + - port: http-envoy-prom + path: /stats/prometheus diff --git a/values/prometheus-operator/prometheus-operator.gotmpl b/values/prometheus-operator/prometheus-operator.gotmpl index 840ec94421..b99a756841 100644 --- a/values/prometheus-operator/prometheus-operator.gotmpl +++ b/values/prometheus-operator/prometheus-operator.gotmpl @@ -55,6 +55,8 @@ prometheusOperator: name: custom-ca kind: ClusterIssuer priorityClassName: "otomi-critical" +commonLabels: + prometheus: system prometheus: enabled: {{ $p.enabled }} namespaceOverride: prometheus @@ -65,6 +67,12 @@ prometheus: podMetadata: annotations: sidecar.istio.io/inject: "true" + podMonitorSelector: + matchLabels: + prometheus: system + serviceMonitorSelector: + matchLabels: + prometheus: system resources: {{- with $p | get "resources" nil }} {{- toYaml . | nindent 6 }} @@ -104,8 +112,18 @@ prometheus: {{- tpl (readFile "../../helmfile.d/snippets/blackbox-targets.gotmpl") (dict "teamId" $teamId "services" $teamServices "domain" $domain) | nindent 6 }} {{- end }} {{- end }} - additionalPodMonitors: {{- readFile "pod-monitors.yaml" | nindent 4 }} - additionalServiceMonitors: {{- readFile "service-monitors.yaml" | nindent 4 }} + additionalPodMonitors: + {{- range $m := (readFile "pod-monitors.yaml" | fromYaml) | get "additionalPodMonitors" }} + - {{- toYaml $m | nindent 6 }} + additionalLabels: + prometheus: system + {{- end }} + additionalServiceMonitors: + {{- range $m := (readFile "service-monitors.yaml" | fromYaml) | get "additionalServiceMonitors" }} + - {{- toYaml $m | nindent 6 }} + additionalLabels: + prometheus: system + {{- end }} {{ if eq $v.cluster.provider "aws" }} additionalPrometheusRules: - name: cluster-autoscaler @@ -181,6 +199,10 @@ grafana: pspEnabled: false podAnnotations: sidecar.istio.io/inject: "true" + serviceMonitor: + namespace: grafana + labels: + prometheus: system testFramework: enabled: false diff --git a/values/prometheus-operator/service-monitors.yaml b/values/prometheus-operator/service-monitors.yaml index f95946bca3..53f92b0707 100644 --- a/values/prometheus-operator/service-monitors.yaml +++ b/values/prometheus-operator/service-monitors.yaml @@ -1,69 +1,59 @@ -- name: blackbox - namespaceSelector: - matchNames: - - monitoring - selector: - matchLabels: - app.kubernetes.io/name: prometheus-blackbox-exporter - endpoints: - - port: http - interval: 30s - path: /metrics -- name: istio-ingressgateway-public - namespace: istio-system - selector: - matchLabels: - istio: ingressgateway-public - namespaceSelector: - matchNames: - - istio-system - endpoints: - - targetPort: http-envoy-prom - path: /stats/prometheus -- name: istio-ingressgateway-private - namespace: istio-system - selector: - matchLabels: - istio: ingressgateway-private - namespaceSelector: - matchNames: - - istio-system - endpoints: - - targetPort: http-envoy-prom - path: /stats/prometheus -- name: istiod - namespace: istio-system - selector: - matchLabels: - istio: pilot - namespaceSelector: - matchNames: - - istio-system - endpoints: - - port: http-monitoring - interval: 15s -- name: jaeger-operator-metrics - namespace: jaeger-operator - selector: - matchLabels: - name: jaeger-operator - namespaceSelector: - matchNames: - - jaeger-operator - endpoints: - - bearerTokenSecret: - key: '' - port: http-metrics - - bearerTokenSecret: - key: '' - port: cr-metrics -- name: argocd-metrics - namespace: argocd - selector: - matchLabels: - app.kubernetes.io/name: argocd-metrics - namespaceSelector: - matchNames: - - argocd - endpoints: - - port: metrics +additionalServiceMonitors: + - name: blackbox + namespaceSelector: + matchNames: + - monitoring + selector: + matchLabels: + app.kubernetes.io/name: prometheus-blackbox-exporter + endpoints: + - port: http + interval: 30s + path: /metrics + - name: istio-ingressgateway-public + namespace: istio-system + selector: + matchLabels: + istio: ingressgateway-public + namespaceSelector: + matchNames: + - istio-system + endpoints: + - targetPort: http-envoy-prom + path: /stats/prometheus + - name: istiod + namespace: istio-system + selector: + matchLabels: + istio: pilot + namespaceSelector: + matchNames: + - istio-system + endpoints: + - port: http-monitoring + interval: 15s + - name: jaeger-operator-metrics + namespace: jaeger-operator + selector: + matchLabels: + name: jaeger-operator + namespaceSelector: + matchNames: + - jaeger-operator + endpoints: + - bearerTokenSecret: + key: '' + port: http-metrics + - bearerTokenSecret: + key: '' + port: cr-metrics + - name: argocd-metrics + namespace: argocd + selector: + matchLabels: + app.kubernetes.io/name: argocd-metrics + namespaceSelector: + matchNames: + - argocd + endpoints: + - port: metrics