SumoLogic · sumo-drosiek · Dec 22, 2022 · Dec 20, 2022 · Dec 20, 2022 · Dec 20, 2022
@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - fix(openshift): changed allowed fsgroups in SecurityContextConstraints [#2717]
 - fix(openshift): set securityContexts for otelcol-logs-collector [#2717]
 - fix: obey proxy settings in otelcol [#2719]
+- feat(metrics): simplify custom application metrics [#2716]
 
 [#2686]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2686
 [#2687]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2687
@@ -22,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 [#2712]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2712
 [#2717]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2717
 [#2719]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2719
+[#2716]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2716
 [Unreleased]: https://github.com/SumoLogic/sumologic-kubernetes-collection/compare/v3.0.0-beta.0...main
 
 ## [v3.0.0-beta.0]

@@ -15,37 +15,7 @@ receivers:
         data_format = "prometheusremotewrite"
         path_tag = true
         paths = [
-          "/prometheus.metrics",
-          "/prometheus.metrics.apiserver",
-          "/prometheus.metrics.applications.activemq",
-          "/prometheus.metrics.applications.apache",
-          "/prometheus.metrics.applications.cassandra",
-          "/prometheus.metrics.applications.couchbase",
-          "/prometheus.metrics.applications.elasticsearch",
-          "/prometheus.metrics.applications.haproxy",
-          "/prometheus.metrics.applications.jmx",
-          "/prometheus.metrics.applications.kafka",
-          "/prometheus.metrics.applications.memcached",
-          "/prometheus.metrics.applications.mongodb",
-          "/prometheus.metrics.applications.mysql",
-          "/prometheus.metrics.applications.nginx",
-          "/prometheus.metrics.applications.nginx-ingress",
-          "/prometheus.metrics.applications.postgresql",
-          "/prometheus.metrics.applications.rabbitmq",
-          "/prometheus.metrics.applications.redis",
-          "/prometheus.metrics.applications.sqlserver",
-          "/prometheus.metrics.applications.squidproxy",
-          "/prometheus.metrics.applications.tomcat",
-          "/prometheus.metrics.applications.varnish",
-          "/prometheus.metrics.container",
-          "/prometheus.metrics.controller-manager",
-          "/prometheus.metrics.control-plane.coredns",
-          "/prometheus.metrics.control-plane.kube-etcd",
-          "/prometheus.metrics.kubelet",
-          "/prometheus.metrics.node",
-          "/prometheus.metrics.operator.rule",
-          "/prometheus.metrics.scheduler",
-          "/prometheus.metrics.state"
+{{ include "metric.remotewrites" . | indent 10 }}
         ]
 extensions:
   health_check: {}

@@ -1549,3 +1549,23 @@ Example Usage:
   value: {{ .Values.sumologic.noProxy }}
 {{- end -}}
 {{- end -}}
+
+
+{{/*
+Generate list of remoteWrite endpoints for telegraf configuration
+
+'{{ include "metric.remotewrites" . }}'
+*/}}
+{{- define "metric.remotewrites" -}}
+{{- $remoteWrites := list -}}
+{{- $kps := get .Values "kube-prometheus-stack" -}}
+{{- range $remoteWrite := $kps.prometheus.prometheusSpec.remoteWrite }}
+{{- $remoteWrites = append $remoteWrites ($remoteWrite.url | trimPrefix "http://$(METADATA_METRICS_SVC).$(NAMESPACE).svc.cluster.local.:9888" | quote) -}}
+{{- end }}
+{{- range $remoteWrite := $kps.prometheus.prometheusSpec.additionalRemoteWrite }}
+{{- $remoteWrites = append $remoteWrites ($remoteWrite.url | trimPrefix "http://$(METADATA_METRICS_SVC).$(NAMESPACE).svc.cluster.local.:9888" | quote) -}}
+{{- end -}}
+{{- $remoteWrites := uniq $remoteWrites -}}
+{{- $remoteWrites := sortAlpha $remoteWrites -}}
+{{ $remoteWrites | join ",\n" }}
+{{- end -}}
@@ -0,0 +1,42 @@
+{{- $kps := get .Values "kube-prometheus-stack" -}}
+{{- if or (and (typeIs "<nil>" $kps.enabled) (.Values.sumologic.metrics.enabled)) (and (typeIs "bool" $kps.enabled) ($kps.enabled)) -}}
+---
+{{- if .Values.sumologic.metrics.serviceMonitors }}
+apiVersion: v1
+kind: List
+items:
+{{- range .Values.sumologic.metrics.serviceMonitors }}
+  - apiVersion: monitoring.coreos.com/v1
+    kind: ServiceMonitor
+    metadata:
+      name: {{ .name }}
+      namespace: {{ template "kube-prometheus-stack.namespace" $ }}
+      labels:
+        app: {{ template "kube-prometheus-stack.name" $ }}-prometheus
+{{ include "kube-prometheus-stack.labels" $ | indent 8 }}
+        {{- if .additionalLabels }}
+{{ toYaml .additionalLabels | indent 8 }}
+        {{- end }}
+    spec:
+      endpoints:
+{{ toYaml .endpoints | indent 8 }}
+    {{- if .jobLabel }}
+      jobLabel: {{ .jobLabel }}
+    {{- end }}
+    {{- if .namespaceSelector }}
+      namespaceSelector:
+{{ toYaml .namespaceSelector | indent 8 }}
+    {{- end }}
+      selector:
+{{ toYaml .selector | indent 8 }}
+    {{- if .targetLabels }}
+      targetLabels:
+{{ toYaml .targetLabels | indent 8 }}
+    {{- end }}
+    {{- if .podTargetLabels }}
+      podTargetLabels:
+{{ toYaml .podTargetLabels | indent 8 }}
+    {{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
@@ -431,6 +431,117 @@ sumologic:
       ## Add custom annotations only to metrics sts pods
       podAnnotations: {}
 
+    ## Prometheus serviceMonitors related to Sumo Logic services
+    ## They are applied only if kube-prometheus-stack is enabled
+    serviceMonitors:
+      - name: collection-sumologic-fluentd-logs
+        additionalLabels:
+          sumologic.com/app: fluentd-logs
+        endpoints:
+          - port: metrics
+        namespaceSelector:
+          matchNames:
+            - $(NAMESPACE)
+        selector:
+          matchLabels:
+            sumologic.com/app: fluentd-logs
+            sumologic.com/scrape: "true"
+      - name: collection-sumologic-otelcol-logs
+        additionalLabels:
+          sumologic.com/app: otelcol-logs
+        endpoints:
+          - port: otelcol-metrics
+        namespaceSelector:
+          matchNames:
+            - $(NAMESPACE)
+        selector:
+          matchLabels:
+            sumologic.com/app: fluentd-logs
+            sumologic.com/scrape: "true"
+      - name: collection-sumologic-fluentd-metrics
+        additionalLabels:
+          sumologic.com/app: fluentd-metrics
+        endpoints:
+          - port: metrics
+        namespaceSelector:
+          matchNames:
+            - $(NAMESPACE)
+        selector:
+          matchLabels:
+            sumologic.com/app: fluentd-metrics
+            sumologic.com/scrape: "true"
+      - name: collection-sumologic-otelcol-metrics
+        additionalLabels:
+          sumologic.com/app: otelcol-metrics
+        endpoints:
+          - port: otelcol-metrics
+        namespaceSelector:
+          matchNames:
+            - $(NAMESPACE)
+        selector:
+          matchLabels:
+            sumologic.com/app: fluentd-metrics
+            sumologic.com/scrape: "true"
+      - name: collection-sumologic-fluentd-events
+        additionalLabels:
+          sumologic.com/app: fluentd-events
+        endpoints:
+          - port: metrics
+        namespaceSelector:
+          matchNames:
+            - $(NAMESPACE)
+        selector:
+          matchLabels:
+            sumologic.com/app: fluentd-events
+            sumologic.com/scrape: "true"
+      - name: collection-fluent-bit
+        additionalLabels:
+          sumologic.com/app: collection-fluent-bit
+        endpoints:
+          - port: http
+            path: /api/v1/metrics/prometheus
+        namespaceSelector:
+          matchNames:
+            - $(NAMESPACE)
+        selector:
+          matchLabels:
+            app.kubernetes.io/name: fluent-bit
+            sumologic.com/scrape: "true"
+      - name: collection-sumologic-otelcol-logs-collector
+        additionalLabels:
+          sumologic.com/app: otelcol-logs-collector
+        endpoints:
+          - port: metrics
+        namespaceSelector:
+          matchNames:
+            - $(NAMESPACE)
+        selector:
+          matchLabels:
+            sumologic.com/app: otelcol-logs-collector
+            sumologic.com/scrape: "true"
+      - name: collection-sumologic-otelcol-traces
+        additionalLabels:
+          sumologic.com/app: otelcol
+        endpoints:
+          - port: metrics
+        namespaceSelector:
+          matchNames:
+            - $(NAMESPACE)
+        selector:
+          matchLabels:
+            sumologic.com/component: traces
+            sumologic.com/scrape: "true"
+      - name: collection-sumologic-prometheus
+        endpoints:
+          - port: web
+            path: /metrics
+        namespaceSelector:
+          matchNames:
+            - $(NAMESPACE)
+        selector:
+          matchLabels:
+            operated-prometheus: "true"
+
   ### Traces configuration
   ## Set the enabled flag to false to disable traces ingestion.
   traces:
@@ -1723,114 +1834,7 @@ kube-prometheus-stack:
       #   cpu: 100m
       #   memory: 30Mi
   prometheus:
-    additionalServiceMonitors:
-      - name: collection-sumologic-fluentd-logs
-        additionalLabels:
-          sumologic.com/app: fluentd-logs
-        endpoints:
-          - port: metrics
-        namespaceSelector:
-          matchNames:
-            - $(NAMESPACE)
-        selector:
-          matchLabels:
-            sumologic.com/app: fluentd-logs
-            sumologic.com/scrape: "true"
-      - name: collection-sumologic-otelcol-logs
-        additionalLabels:
-          sumologic.com/app: otelcol-logs
-        endpoints:
-          - port: otelcol-metrics
-        namespaceSelector:
-          matchNames:
-            - $(NAMESPACE)
-        selector:
-          matchLabels:
-            sumologic.com/app: fluentd-logs
-            sumologic.com/scrape: "true"
-      - name: collection-sumologic-fluentd-metrics
-        additionalLabels:
-          sumologic.com/app: fluentd-metrics
-        endpoints:
-          - port: metrics
-        namespaceSelector:
-          matchNames:
-            - $(NAMESPACE)
-        selector:
-          matchLabels:
-            sumologic.com/app: fluentd-metrics
-            sumologic.com/scrape: "true"
-      - name: collection-sumologic-otelcol-metrics
-        additionalLabels:
-          sumologic.com/app: otelcol-metrics
-        endpoints:
-          - port: otelcol-metrics
-        namespaceSelector:
-          matchNames:
-            - $(NAMESPACE)
-        selector:
-          matchLabels:
-            sumologic.com/app: fluentd-metrics
-            sumologic.com/scrape: "true"
-      - name: collection-sumologic-fluentd-events
-        additionalLabels:
-          sumologic.com/app: fluentd-events
-        endpoints:
-          - port: metrics
-        namespaceSelector:
-          matchNames:
-            - $(NAMESPACE)
-        selector:
-          matchLabels:
-            sumologic.com/app: fluentd-events
-            sumologic.com/scrape: "true"
-      - name: collection-fluent-bit
-        additionalLabels:
-          sumologic.com/app: collection-fluent-bit
-        endpoints:
-          - port: http
-            path: /api/v1/metrics/prometheus
-        namespaceSelector:
-          matchNames:
-            - $(NAMESPACE)
-        selector:
-          matchLabels:
-            app.kubernetes.io/name: fluent-bit
-            sumologic.com/scrape: "true"
-      - name: collection-sumologic-otelcol-logs-collector
-        additionalLabels:
-          sumologic.com/app: otelcol-logs-collector
-        endpoints:
-          - port: metrics
-        namespaceSelector:
-          matchNames:
-            - $(NAMESPACE)
-        selector:
-          matchLabels:
-            sumologic.com/app: otelcol-logs-collector
-            sumologic.com/scrape: "true"
-      - name: collection-sumologic-otelcol-traces
-        additionalLabels:
-          sumologic.com/app: otelcol
-        endpoints:
-          - port: metrics
-        namespaceSelector:
-          matchNames:
-            - $(NAMESPACE)
-        selector:
-          matchLabels:
-            sumologic.com/component: traces
-            sumologic.com/scrape: "true"
-      - name: collection-sumologic-prometheus
-        endpoints:
-          - port: web
-            path: /metrics
-        namespaceSelector:
-          matchNames:
-            - $(NAMESPACE)
-        selector:
-          matchLabels:
-            operated-prometheus: "true"
+    additionalServiceMonitors: []
     prometheusSpec:
       ## Prometheus default scrape interval, default from upstream Kube Prometheus Stack Helm chart
       ## NOTE changing the scrape interval to be >1m can result in metrics
@@ -1939,6 +1943,13 @@ kube-prometheus-stack:
               target_label: pod
               replacement: $1
               action: replace
+            ## Sets _sumo_forward_ label to true
+            - source_labels: [__name__]
+              separator: ;
+              regex: (.*)
+              target_label: _sumo_forward_
+              replacement: "true"
+              action: replace
             - source_labels: [__meta_kubernetes_service_name]
               separator: ;
               regex: (.*)
@@ -3315,7 +3326,16 @@ kube-prometheus-stack:
               sourceLabels: [__name__]
 
       ## additionalRemoteWrite is appended to remoteWrite
-      additionalRemoteWrite: []
+      additionalRemoteWrite:
+        ## Forward every metric which has _sumo_forward_ label set to true
+        - url: http://$(METADATA_METRICS_SVC).$(NAMESPACE).svc.cluster.local.:9888/prometheus.metrics.applications.custom
+          remoteTimeout: 5s
+          writeRelabelConfigs:
+            - action: keep
+              regex: ^true$
+              sourceLabels: [_sumo_forward_]
+            - action: labeldrop
+              regex: _sumo_forward_
 
       serviceMonitor:
         selfMonitor: false