fix: servicemonitor labels (#822)

also fixed loki regression by adding retention enabled flag, removed unnecessary monitor

binzx/otomi

@@ -27,7 +27,7 @@ elif [ -z "$IN_DOCKER" ] && [ -n "$ENV_DIR" ]; then
 fi
 [[ "$ENV_DIR" == *"../"* ]] && echo "Don't provide an ENV_DIR that contains '../'!" && exit 1
 # treat CI and chart as non interactive
-if [ -z "$PS1" ]; then
+if [[ $calling_args != 'bash '* ]] && [[ $calling_args != 'x '* ]] && { [ -z "$PS1" ] || [ -n "$CI" ]; }; then
   calling_args="$calling_args --non-interactive"
 fi
 if [[ $calling_args == 'x '* ]] && [[ $calling_args != 'x -- '* ]]; then

charts/prometheus-operator/charts/grafana/templates/servicemonitor.yaml

@@ -38,5 +38,5 @@ spec:
       {{- include "grafana.selectorLabels" . | nindent 8 }}
   namespaceSelector:
     matchNames:
-      - {{ .Release.Namespace }}
+      - {{ template "grafana.namespace" . }}
 {{- end }}

helmfile.d/helmfile-60.teams.yaml

@@ -81,10 +81,18 @@ releases:
             alertmanager: false
         enabled: false
         {{- end }}
+        commonLabels:
+          prometheus: team-{{ $teamId }}
         prometheus:
           namespaceOverride: team-{{ $teamId }}
           prometheusSpec:
             externalUrl: https://{{ $appsDomain }}/prometheus
+            podMonitorSelector:
+              matchLabels:
+                prometheus: team-{{ $teamId }}
+            serviceMonitorSelector:
+              matchLabels:
+                prometheus: team-{{ $teamId }}
             ruleNamespaceSelector:
               matchLabels:
                 name: team-{{ $teamId }}

values/loki/loki.gotmpl

@@ -44,6 +44,7 @@ config:
     {{ $l.storageType }}: {{- $l | get $l.storageType | toYaml | nindent 6 }}
     {{- end }}
   compactor:
+    retention_enabled: true
     shared_store: {{ $l | get "storageType" "filesystem" }}
   chunk_store_config:
     max_look_back_period: {{ $l | get "retention.period" "24h" }}

values/prometheus-operator/pod-monitors.yaml

@@ -1,8 +1,10 @@
-- name: istio-sidecars
-  namespace: istio-system
-  selector:
-    matchLabels:
-      security.istio.io/tlsMode: istio
-  podMetricsEndpoints:
-    - port: http-envoy-prom
-      path: /stats/prometheus
+additionalPodMonitors:
+  - name: istio-sidecars
+    selector:
+      matchLabels:
+        security.istio.io/tlsMode: istio
+    namespaceSelector:
+      any: true
+    podMetricsEndpoints:
+      - port: http-envoy-prom
+        path: /stats/prometheus

values/prometheus-operator/prometheus-operator.gotmpl

@@ -55,6 +55,8 @@ prometheusOperator:
         name: custom-ca
         kind: ClusterIssuer
   priorityClassName: "otomi-critical"
+commonLabels:
+  prometheus: system
 prometheus:
   enabled: {{ $p.enabled }}
   namespaceOverride: prometheus
@@ -65,6 +67,12 @@ prometheus:
     podMetadata:
       annotations:
         sidecar.istio.io/inject: "true"
+    podMonitorSelector:
+      matchLabels:
+        prometheus: system
+    serviceMonitorSelector:
+      matchLabels:
+        prometheus: system
     resources:
       {{- with $p | get "resources" nil }}
         {{- toYaml . | nindent 6 }}
@@ -104,8 +112,18 @@ prometheus:
     {{- tpl (readFile "../../helmfile.d/snippets/blackbox-targets.gotmpl") (dict "teamId" $teamId "services" $teamServices "domain" $domain) | nindent 6 }}
   {{- end }}
 {{- end }}
-  additionalPodMonitors: {{- readFile "pod-monitors.yaml" | nindent 4 }}
-  additionalServiceMonitors: {{- readFile "service-monitors.yaml" | nindent 4 }}
+  additionalPodMonitors:
+    {{- range $m := (readFile "pod-monitors.yaml" | fromYaml) | get "additionalPodMonitors" }}
+    - {{- toYaml $m | nindent 6 }}
+      additionalLabels:
+        prometheus: system
+    {{- end }}
+  additionalServiceMonitors:
+    {{- range $m := (readFile "service-monitors.yaml" | fromYaml) | get "additionalServiceMonitors" }}
+    - {{- toYaml $m | nindent 6 }}
+      additionalLabels:
+        prometheus: system
+    {{- end }}
 {{ if eq $v.cluster.provider "aws" }}  
 additionalPrometheusRules:
   - name: cluster-autoscaler
@@ -181,6 +199,10 @@ grafana:
     pspEnabled: false
   podAnnotations:
     sidecar.istio.io/inject: "true"
+  serviceMonitor:
+    namespace: grafana
+    labels:
+      prometheus: system
 
   testFramework:
     enabled: false

values/prometheus-operator/service-monitors.yaml

@@ -1,69 +1,59 @@
-- name: blackbox
-  namespaceSelector:
-    matchNames:
-      - monitoring
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: prometheus-blackbox-exporter
-  endpoints:
-    - port: http
-      interval: 30s
-      path: /metrics
-- name: istio-ingressgateway-public
-  namespace: istio-system
-  selector:
-    matchLabels:
-      istio: ingressgateway-public
-  namespaceSelector:
-    matchNames:
-      - istio-system
-  endpoints:
-    - targetPort: http-envoy-prom
-      path: /stats/prometheus
-- name: istio-ingressgateway-private
-  namespace: istio-system
-  selector:
-    matchLabels:
-      istio: ingressgateway-private
-  namespaceSelector:
-    matchNames:
-      - istio-system
-  endpoints:
-    - targetPort: http-envoy-prom
-      path: /stats/prometheus
-- name: istiod
-  namespace: istio-system
-  selector:
-    matchLabels:
-      istio: pilot
-  namespaceSelector:
-    matchNames:
-      - istio-system
-  endpoints:
-    - port: http-monitoring
-      interval: 15s
-- name: jaeger-operator-metrics
-  namespace: jaeger-operator
-  selector:
-    matchLabels:
-      name: jaeger-operator
-  namespaceSelector:
-    matchNames:
-      - jaeger-operator
-  endpoints:
-    - bearerTokenSecret:
-        key: ''
-      port: http-metrics
-    - bearerTokenSecret:
-        key: ''
-      port: cr-metrics
-- name: argocd-metrics
-  namespace: argocd
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: argocd-metrics
-  namespaceSelector:
-    matchNames:
-      - argocd
-  endpoints:
-    - port: metrics
+additionalServiceMonitors:
+  - name: blackbox
+    namespaceSelector:
+      matchNames:
+        - monitoring
+    selector:
+      matchLabels:
+        app.kubernetes.io/name: prometheus-blackbox-exporter
+    endpoints:
+      - port: http
+        interval: 30s
+        path: /metrics
+  - name: istio-ingressgateway-public
+    namespace: istio-system
+    selector:
+      matchLabels:
+        istio: ingressgateway-public
+    namespaceSelector:
+      matchNames:
+        - istio-system
+    endpoints:
+      - targetPort: http-envoy-prom
+        path: /stats/prometheus
+  - name: istiod
+    namespace: istio-system
+    selector:
+      matchLabels:
+        istio: pilot
+    namespaceSelector:
+      matchNames:
+        - istio-system
+    endpoints:
+      - port: http-monitoring
+        interval: 15s
+  - name: jaeger-operator-metrics
+    namespace: jaeger-operator
+    selector:
+      matchLabels:
+        name: jaeger-operator
+    namespaceSelector:
+      matchNames:
+        - jaeger-operator
+    endpoints:
+      - bearerTokenSecret:
+          key: ''
+        port: http-metrics
+      - bearerTokenSecret:
+          key: ''
+        port: cr-metrics
+  - name: argocd-metrics
+    namespace: argocd
+    selector:
+      matchLabels:
+        app.kubernetes.io/name: argocd-metrics
+    namespaceSelector:
+      matchNames:
+        - argocd
+    endpoints:
+      - port: metrics