From 04b298dc4fa95ebfa2acc74b0343dc9e3fe570ea Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Wed, 24 Apr 2024 12:55:35 -0600 Subject: [PATCH 1/4] chore: refactor promtail extraScrapeConfigs into scrapeConfigs --- src/promtail/values/values.yaml | 43 ++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/src/promtail/values/values.yaml b/src/promtail/values/values.yaml index da8124aad..65f5cd520 100644 --- a/src/promtail/values/values.yaml +++ b/src/promtail/values/values.yaml @@ -3,7 +3,48 @@ config: - url: 'http://loki-gateway.loki.svc.cluster.local:80/loki/api/v1/push' snippets: - extraScrapeConfigs: > + scrapeConfigs: | + # From upstream chart https://github.com/grafana/helm-charts/blob/main/charts/promtail/values.yaml + # See also https://github.com/grafana/loki/blob/master/production/ksonnet/promtail/scrape_config.libsonnet for reference + - job_name: kubernetes-pods + pipeline_stages: + {{- toYaml .Values.config.snippets.pipelineStages | nindent 4 }} + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: + - __meta_kubernetes_pod_controller_name + regex: ([0-9a-z-.]+?)(-[0-9a-f]{8,10})? + action: replace + target_label: __tmp_controller_name + - source_labels: + - __meta_kubernetes_pod_label_app_kubernetes_io_name + - __meta_kubernetes_pod_label_app + - __tmp_controller_name + - __meta_kubernetes_pod_name + regex: ^;*([^;]+)(;.*)?$ + action: replace + target_label: app + - source_labels: + - __meta_kubernetes_pod_label_app_kubernetes_io_instance + - __meta_kubernetes_pod_label_instance + regex: ^;*([^;]+)(;.*)?$ + action: replace + target_label: instance + - source_labels: + - __meta_kubernetes_pod_label_app_kubernetes_io_component + - __meta_kubernetes_pod_label_component + regex: ^;*([^;]+)(;.*)?$ + action: replace + target_label: component + {{- if .Values.config.snippets.addScrapeJobLabel }} + - replacement: kubernetes-pods + target_label: scrape_job + {{- end }} + {{- toYaml .Values.config.snippets.common | nindent 4 }} + {{- with .Values.config.snippets.extraRelabelConfigs }} + {{- toYaml . | nindent 4 }} + {{- end }} - job_name: systemd-messages static_configs: - targets: [localhost] From 12a7b0ff5d3e09a7ceccaef70428dae91d8d60e0 Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Thu, 25 Apr 2024 08:40:16 -0600 Subject: [PATCH 2/4] separate upstream default from core default --- src/promtail/values/values.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/promtail/values/values.yaml b/src/promtail/values/values.yaml index 65f5cd520..a64d8864d 100644 --- a/src/promtail/values/values.yaml +++ b/src/promtail/values/values.yaml @@ -4,7 +4,7 @@ config: snippets: scrapeConfigs: | - # From upstream chart https://github.com/grafana/helm-charts/blob/main/charts/promtail/values.yaml + # Upstream Defaults https://github.com/grafana/helm-charts/blob/main/charts/promtail/values.yaml # See also https://github.com/grafana/loki/blob/master/production/ksonnet/promtail/scrape_config.libsonnet for reference - job_name: kubernetes-pods pipeline_stages: @@ -45,6 +45,7 @@ config: {{- with .Values.config.snippets.extraRelabelConfigs }} {{- toYaml . | nindent 4 }} {{- end }} + # UDS CORE Defaults - job_name: systemd-messages static_configs: - targets: [localhost] From 468895e768fe1751991d6d99435c20855f5410a7 Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Fri, 26 Apr 2024 10:29:29 -0600 Subject: [PATCH 3/4] set scrape_config job specifically for log files under /var/log/kubernetes --- src/promtail/values/values.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/promtail/values/values.yaml b/src/promtail/values/values.yaml index a64d8864d..0e9f7cf09 100644 --- a/src/promtail/values/values.yaml +++ b/src/promtail/values/values.yaml @@ -63,6 +63,13 @@ config: - source_labels: - __journal_syslog_identifier target_label: syslog_identifier + - job_name: kubernetes-logs + static_configs: + - targets: [localhost] + labels: + job: kubernetes-logs + host: "${NODE_HOSTNAME}" + __path__: /var/log/kubernetes/**/*.log containerSecurityContext: allowPrivilegeEscalation: false @@ -90,6 +97,9 @@ extraVolumes: - hostPath: path: /etc name: machine-id + - hostPath: + path: /var/lib/rancher/rke2/server/logs/audit.log + name: audit-logs extraVolumeMounts: - mountPath: /var/log @@ -99,6 +109,9 @@ extraVolumeMounts: name: machine-id readOnly: true subPath: machine-id + - mountPath: /var/lib/rancher/rke2/server/logs/audit.log + name: audit-logs + readOnly: true resources: limits: From ec489e0f61bb4cb09266411e8474a38f5d00253c Mon Sep 17 00:00:00 2001 From: TristanHoladay <40547442+TristanHoladay@users.noreply.github.com> Date: Fri, 26 Apr 2024 11:29:37 -0600 Subject: [PATCH 4/4] remove hostpath for auit log path --- src/promtail/values/values.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/promtail/values/values.yaml b/src/promtail/values/values.yaml index 9f91f6122..d7bb9af71 100644 --- a/src/promtail/values/values.yaml +++ b/src/promtail/values/values.yaml @@ -97,9 +97,6 @@ extraVolumes: - hostPath: path: /etc name: machine-id - - hostPath: - path: /var/lib/rancher/rke2/server/logs/audit.log - name: audit-logs extraVolumeMounts: - mountPath: /var/log @@ -109,9 +106,6 @@ extraVolumeMounts: name: machine-id readOnly: true subPath: machine-id - - mountPath: /var/lib/rancher/rke2/server/logs/audit.log - name: audit-logs - readOnly: true resources: limits: