From e29edcad6c077ccf9e6e21a3074a55ebf003fcb2 Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Fri, 9 Apr 2021 14:31:29 +0200 Subject: [PATCH 01/37] introduce sumologic.logs.provider Signed-off-by: Dominik Rosiek --- deploy/helm/sumologic/templates/_helpers.tpl | 6 +++++- deploy/helm/sumologic/templates/logs/fluentd/configmap.yaml | 2 +- .../sumologic/templates/logs/fluentd/service-headless.yaml | 2 +- deploy/helm/sumologic/templates/logs/fluentd/service.yaml | 2 +- .../helm/sumologic/templates/logs/fluentd/statefulset.yaml | 2 +- deploy/helm/sumologic/values.yaml | 2 ++ 6 files changed, 11 insertions(+), 5 deletions(-) diff --git a/deploy/helm/sumologic/templates/_helpers.tpl b/deploy/helm/sumologic/templates/_helpers.tpl index 536f2a17db..8123920020 100644 --- a/deploy/helm/sumologic/templates/_helpers.tpl +++ b/deploy/helm/sumologic/templates/_helpers.tpl @@ -297,7 +297,11 @@ helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded {{- end -}} {{- define "sumologic.metadata.name.logs" -}} -{{ template "sumologic.metadata.name.fluentd" . }}-logs +{{- if eq .Values.sumologic.logs.provider "fluentd" -}} +template "sumologic.metadata.name.fluentd" . }}-logs +{{- else if eq .Values.sumologic.logs.provider "otelcol" -}} +template "sumologic.metadata.name.otelcol" . }}-logs +{{- end -}} {{- end -}} {{- define "sumologic.metadata.name.logs.service" -}} diff --git a/deploy/helm/sumologic/templates/logs/fluentd/configmap.yaml b/deploy/helm/sumologic/templates/logs/fluentd/configmap.yaml index 8f9bff3ef0..20653a8162 100644 --- a/deploy/helm/sumologic/templates/logs/fluentd/configmap.yaml +++ b/deploy/helm/sumologic/templates/logs/fluentd/configmap.yaml @@ -1,4 +1,4 @@ -{{- if and (eq .Values.sumologic.logs.enabled true) (eq .Values.fluentd.logs.enabled true) }} +{{- if and (eq .Values.sumologic.logs.enabled true) (eq .Values.sumologic.logs.provider "fluentd") (eq .Values.fluentd.logs.enabled true) }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/deploy/helm/sumologic/templates/logs/fluentd/service-headless.yaml b/deploy/helm/sumologic/templates/logs/fluentd/service-headless.yaml index d24ae81c85..266b180731 100644 --- a/deploy/helm/sumologic/templates/logs/fluentd/service-headless.yaml +++ b/deploy/helm/sumologic/templates/logs/fluentd/service-headless.yaml @@ -1,4 +1,4 @@ -{{- if and (eq .Values.sumologic.logs.enabled true) (eq .Values.fluentd.logs.enabled true) }} +{{- if and (eq .Values.sumologic.logs.enabled true) (eq .Values.sumologic.logs.provider "fluentd") (eq .Values.fluentd.logs.enabled true) }} apiVersion: v1 kind: Service metadata: diff --git a/deploy/helm/sumologic/templates/logs/fluentd/service.yaml b/deploy/helm/sumologic/templates/logs/fluentd/service.yaml index e523fd92e5..b30100eceb 100644 --- a/deploy/helm/sumologic/templates/logs/fluentd/service.yaml +++ b/deploy/helm/sumologic/templates/logs/fluentd/service.yaml @@ -1,4 +1,4 @@ -{{- if and (eq .Values.sumologic.logs.enabled true) (eq .Values.fluentd.logs.enabled true) }} +{{- if and (eq .Values.sumologic.logs.enabled true) (eq .Values.sumologic.logs.provider "fluentd") (eq .Values.fluentd.logs.enabled true) }} apiVersion: v1 kind: Service metadata: diff --git a/deploy/helm/sumologic/templates/logs/fluentd/statefulset.yaml b/deploy/helm/sumologic/templates/logs/fluentd/statefulset.yaml index 3775c0084f..2e718e4e36 100644 --- a/deploy/helm/sumologic/templates/logs/fluentd/statefulset.yaml +++ b/deploy/helm/sumologic/templates/logs/fluentd/statefulset.yaml @@ -1,4 +1,4 @@ -{{- if and (eq .Values.sumologic.logs.enabled true) (eq .Values.fluentd.logs.enabled true) }} +{{- if and (eq .Values.sumologic.logs.enabled true) (eq .Values.sumologic.logs.provider "fluentd") (eq .Values.fluentd.logs.enabled true) }} apiVersion: apps/v1 kind: StatefulSet metadata: diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index c7eea49a55..4e200c4737 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -183,6 +183,8 @@ sumologic: ## Set the enabled flag to false for disabling logs ingestion altogether. logs: enabled: true + ## Set provider service (either fluentd or otelcol) + provider: fluentd ## Fields to be created at Sumo Logic to ensure logs are tagged with ## relevant metadata. From a9336d390f977d77c792911c175e4dec56718670 Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Mon, 17 May 2021 11:01:44 +0200 Subject: [PATCH 02/37] feat(helm): add otelcol logs provider (statefulset) --- .../sumologic/conf/logs/otelcol/config.yaml | 1 + deploy/helm/sumologic/templates/_helpers.tpl | 4 +- .../templates/logs/otelcol/configmap.yaml | 11 ++ .../templates/logs/otelcol/statefulset.yaml | 149 ++++++++++++++++++ deploy/helm/sumologic/values.yaml | 29 ++++ 5 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 deploy/helm/sumologic/conf/logs/otelcol/config.yaml create mode 100644 deploy/helm/sumologic/templates/logs/otelcol/configmap.yaml create mode 100644 deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml diff --git a/deploy/helm/sumologic/conf/logs/otelcol/config.yaml b/deploy/helm/sumologic/conf/logs/otelcol/config.yaml new file mode 100644 index 0000000000..b8244d90f6 --- /dev/null +++ b/deploy/helm/sumologic/conf/logs/otelcol/config.yaml @@ -0,0 +1 @@ +{{ tpl (toYaml .Values.otelcol.logs.config | replace ": '{{" ": {{" | replace "}}'" "}}") . | nindent 2 }} diff --git a/deploy/helm/sumologic/templates/_helpers.tpl b/deploy/helm/sumologic/templates/_helpers.tpl index 8123920020..797cb2ea25 100644 --- a/deploy/helm/sumologic/templates/_helpers.tpl +++ b/deploy/helm/sumologic/templates/_helpers.tpl @@ -298,9 +298,9 @@ helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded {{- define "sumologic.metadata.name.logs" -}} {{- if eq .Values.sumologic.logs.provider "fluentd" -}} -template "sumologic.metadata.name.fluentd" . }}-logs +{{ template "sumologic.metadata.name.fluentd" . }}-logs {{- else if eq .Values.sumologic.logs.provider "otelcol" -}} -template "sumologic.metadata.name.otelcol" . }}-logs +{{ template "sumologic.metadata.name.otelcol" . }}-logs {{- end -}} {{- end -}} diff --git a/deploy/helm/sumologic/templates/logs/otelcol/configmap.yaml b/deploy/helm/sumologic/templates/logs/otelcol/configmap.yaml new file mode 100644 index 0000000000..cacc706e5b --- /dev/null +++ b/deploy/helm/sumologic/templates/logs/otelcol/configmap.yaml @@ -0,0 +1,11 @@ +{{- if and (eq .Values.sumologic.logs.enabled true) (eq .Values.sumologic.logs.provider "otelcol") (eq .Values.otelcol.logs.enabled true) }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "sumologic.metadata.name.logs.configmap" . }} + labels: + app: {{ template "sumologic.labels.app.logs.configmap" . }} + {{- include "sumologic.labels.common" . | nindent 4 }} +data: + {{- (tpl (.Files.Glob "conf/logs/otelcol/config.yaml").AsConfig .) | nindent 2 }} +{{- end }} diff --git a/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml b/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml new file mode 100644 index 0000000000..59f956f28d --- /dev/null +++ b/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml @@ -0,0 +1,149 @@ +{{- if and (eq .Values.sumologic.logs.enabled true) (eq .Values.sumologic.logs.provider "otelcol") (eq .Values.otelcol.logs.enabled true) }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ template "sumologic.metadata.name.logs.statefulset" . }} + labels: + app: {{ template "sumologic.labels.app.logs.statefulset" . }} + {{- include "sumologic.labels.common" . | nindent 4 }} +spec: + selector: + matchLabels: + app: {{ template "sumologic.labels.app.logs.pod" . }} + serviceName: {{ template "sumologic.metadata.name.logs.service-headless" . }} + podManagementPolicy: "Parallel" +{{- if not .Values.fluentd.logs.autoscaling.enabled }} + replicas: {{ .Values.fluentd.logs.statefulset.replicaCount }} +{{- end }} + template: + metadata: + annotations: + checksum/config: {{ include (print $.Template.BasePath "/logs/otelcol/configmap.yaml") . | sha256sum }} +{{- if .Values.sumologic.podAnnotations }} +{{ toYaml .Values.sumologic.podAnnotations | indent 8 }} +{{- end }} +{{- if .Values.fluentd.podAnnotations }} +{{ toYaml .Values.fluentd.podAnnotations | indent 8 }} +{{- end }} +{{- if .Values.fluentd.logs.statefulset.podAnnotations }} +{{ toYaml .Values.fluentd.logs.statefulset.podAnnotations | indent 8 }} +{{- end }} + labels: + app: {{ template "sumologic.labels.app.logs.pod" . }} + {{- include "sumologic.labels.common" . | nindent 8 }} +{{- if .Values.sumologic.podLabels }} +{{ toYaml .Values.sumologic.podLabels | indent 8 }} +{{- end }} +{{- if .Values.fluentd.podLabels }} +{{ toYaml .Values.fluentd.podLabels | indent 8 }} +{{- end }} +{{- if .Values.fluentd.logs.statefulset.podLabels }} +{{ toYaml .Values.fluentd.logs.statefulset.podLabels | indent 8 }} +{{- end }} + spec: + serviceAccountName: {{ template "sumologic.metadata.name.roles.serviceaccount" . }} +{{- if .Values.fluentd.logs.statefulset.nodeSelector }} + nodeSelector: +{{ toYaml .Values.fluentd.logs.statefulset.nodeSelector | indent 8 }} +{{- end }} +{{- if .Values.fluentd.logs.statefulset.tolerations }} + tolerations: +{{ toYaml .Values.fluentd.logs.statefulset.tolerations | indent 8 }} +{{- end }} +{{- if or .Values.fluentd.logs.statefulset.affinity .Values.fluentd.logs.statefulset.podAntiAffinity }} + affinity: +{{- if .Values.fluentd.logs.statefulset.affinity }} +{{ toYaml .Values.fluentd.logs.statefulset.affinity | indent 8 }} +{{- end }} +{{- if eq .Values.fluentd.logs.statefulset.podAntiAffinity "hard"}} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - {{ template "sumologic.labels.app.logs" . }} + - {{ template "sumologic.labels.app.metrics" . }} + - key: app + operator: In + values: + - prometheus-operator-prometheus + topologyKey: "kubernetes.io/hostname" +{{else if eq .Values.fluentd.logs.statefulset.podAntiAffinity "soft"}} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - {{ template "sumologic.labels.app.logs.pod" . }} + - {{ template "sumologic.labels.app.metrics.pod" . }} + - key: app + operator: In + values: + - prometheus-operator-prometheus + topologyKey: "kubernetes.io/hostname" +{{- end }} +{{- end }} + volumes: + - name: config-volume + configMap: + name: {{ template "sumologic.metadata.name.logs.configmap" . }} +{{- if .Values.fluentd.logs.extraVolumes }} +{{ toYaml .Values.fluentd.logs.extraVolumes | indent 6 }} +{{- end }} + securityContext: + {{- toYaml .Values.fluentd.securityContext | nindent 8 }} + {{- if .Values.fluentd.logs.statefulset.priorityClassName }} + priorityClassName: {{ .Values.fluentd.logs.statefulset.priorityClassName | quote }} + {{- end }} + containers: + - name: otelcol + image: {{ .Values.otelcol.statefulset.image.repository }}:{{ .Values.otelcol.statefulset.image.tag }} + imagePullPolicy: {{ .Values.otelcol.statefulset.image.pullPolicy }} + resources: + {{- toYaml .Values.fluentd.logs.statefulset.resources | nindent 10 }} + {{- if .Values.fluentd.logs.statefulset.containers.fluentd.securityContext }} + securityContext: + {{- toYaml .Values.fluentd.otelcol.statefulset.containers.otelcol.securityContext | nindent 10 }} + {{- end }} + ports: + {{- if .Values.sumologic.traces.enabled }} + - name: zipkin-write + containerPort: 9411 + protocol: TCP + {{- end }} + - name: fluent-bit + containerPort: 24321 + protocol: TCP + livenessProbe: + httpGet: + path: / + port: 13133 # Health Check extension default port. + readinessProbe: + httpGet: + path: / + port: 13133 # Health Check extension default port. + volumeMounts: + - name: config-volume + mountPath: /etc/otel/config.yaml + subPath: config.yaml +{{- if .Values.fluentd.logs.extraVolumeMounts }} +{{ toYaml .Values.fluentd.logs.extraVolumeMounts | indent 8 }} +{{- end }} + env: +{{- $ctx := .Values -}} +{{ include "kubernetes.sources.envs" (dict "Context" $ctx "Type" "logs")}} +{{- if .Values.sumologic.traces.enabled }} +{{ include "kubernetes.sources.envs" (dict "Context" $ctx "Type" "traces")}} +{{- end }} + {{- if .Values.fluentd.logs.extraEnvVars }} +{{ toYaml .Values.fluentd.logs.extraEnvVars | nindent 8 }} + {{- end }} + - name: ADDITIONAL_PLUGINS + value: {{ join " " .Values.fluentd.additionalPlugins | quote }} +{{- end }} diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 4e200c4737..758705fe12 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -27,6 +27,7 @@ sumologic: ## Sumo API endpoint; Leave blank for automatic endpoint discovery and redirection ## ref: https://help.sumologic.com/APIs/General-API-Information/Sumo-Logic-Endpoints-and-Firewall-Security endpoint: "" + open_endpoint: "" ## proxy urls httpProxy: "" @@ -2618,6 +2619,34 @@ otelcol: exporters: - sumologic + logs: + enabled: true + config: + receivers: + fluentforward: + endpoint: 0.0.0.0:24321 + extensions: + health_check: {} + # sumologic: + # access_id: '{{ .Values.sumologic.accessId }}' + # access_key: '{{ .Values.sumologic.accessKey }}' + # collector_name: otc-{{ .Values.sumologic.collectorName | default .Values.sumologic.clusterName}} + # api_base_url: '{{ .Values.sumologic.open_endpoint }}' + exporters: + sumologic: + log_format: json + endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE} + service: + extensions: + - health_check + # - sumologic + pipelines: + logs: + receivers: + - fluentforward + processors: [] + exporters: + - sumologic deployment: nodeSelector: {} tolerations: [] From d3f3b91c9ccd6b10d3d635d699d2552782873383 Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Thu, 15 Jul 2021 12:54:34 +0200 Subject: [PATCH 03/37] feat(helm,logs): use fluentds hpa, services, pdb by otelcol Signed-off-by: Dominik Rosiek --- deploy/helm/sumologic/templates/_helpers.tpl | 20 +++++++++++++++++++ .../sumologic/templates/logs/fluentd/hpa.yaml | 2 +- .../sumologic/templates/logs/fluentd/pdb.yaml | 2 +- .../logs/fluentd/service-headless.yaml | 2 +- .../templates/logs/fluentd/service.yaml | 2 +- 5 files changed, 24 insertions(+), 4 deletions(-) diff --git a/deploy/helm/sumologic/templates/_helpers.tpl b/deploy/helm/sumologic/templates/_helpers.tpl index 797cb2ea25..263cfbe442 100644 --- a/deploy/helm/sumologic/templates/_helpers.tpl +++ b/deploy/helm/sumologic/templates/_helpers.tpl @@ -1085,6 +1085,26 @@ Example Usage: {{ $enabled }} {{- end -}} +{{/* +Check if any logs provider is enabled + +Example Usage: +{{- if eq (include "logs.enabled" .) "true" }} + +*/}} +{{- define "logs.enabled" -}} +{{- $enabled := false -}} +{{- if eq .Values.sumologic.logs.enabled true -}} +{{- if and (eq .Values.sumologic.logs.provider "fluentd") (eq .Values.fluentd.logs.enabled true) -}} +{{- $enabled = true -}} +{{- end -}} +{{- if and (eq .Values.sumologic.logs.provider "otelcol") (eq .Values.otelcol.logs.enabled true) -}} +{{- $enabled = true -}} +{{- end -}} +{{- end -}} +{{ $enabled }} +{{- end -}} + {{/* Add service labels diff --git a/deploy/helm/sumologic/templates/logs/fluentd/hpa.yaml b/deploy/helm/sumologic/templates/logs/fluentd/hpa.yaml index da2e135d91..883eb4cbc6 100644 --- a/deploy/helm/sumologic/templates/logs/fluentd/hpa.yaml +++ b/deploy/helm/sumologic/templates/logs/fluentd/hpa.yaml @@ -1,4 +1,4 @@ -{{- if .Values.fluentd.logs.autoscaling.enabled}} +{{- if eq (include "logs.enabled" .) "true" }} apiVersion: autoscaling/v2beta2 kind: HorizontalPodAutoscaler metadata: diff --git a/deploy/helm/sumologic/templates/logs/fluentd/pdb.yaml b/deploy/helm/sumologic/templates/logs/fluentd/pdb.yaml index cca3255c11..29771c5b69 100644 --- a/deploy/helm/sumologic/templates/logs/fluentd/pdb.yaml +++ b/deploy/helm/sumologic/templates/logs/fluentd/pdb.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.sumologic.logs.enabled .Values.fluentd.logs.enabled -}} +{{- if eq (include "logs.enabled" .) "true" }} {{- if .Values.fluentd.logs.podDisruptionBudget -}} apiVersion: policy/v1beta1 kind: PodDisruptionBudget diff --git a/deploy/helm/sumologic/templates/logs/fluentd/service-headless.yaml b/deploy/helm/sumologic/templates/logs/fluentd/service-headless.yaml index 266b180731..84b1a81361 100644 --- a/deploy/helm/sumologic/templates/logs/fluentd/service-headless.yaml +++ b/deploy/helm/sumologic/templates/logs/fluentd/service-headless.yaml @@ -1,4 +1,4 @@ -{{- if and (eq .Values.sumologic.logs.enabled true) (eq .Values.sumologic.logs.provider "fluentd") (eq .Values.fluentd.logs.enabled true) }} +{{- if eq (include "logs.enabled" .) "true" }} apiVersion: v1 kind: Service metadata: diff --git a/deploy/helm/sumologic/templates/logs/fluentd/service.yaml b/deploy/helm/sumologic/templates/logs/fluentd/service.yaml index b30100eceb..b79ea8fb15 100644 --- a/deploy/helm/sumologic/templates/logs/fluentd/service.yaml +++ b/deploy/helm/sumologic/templates/logs/fluentd/service.yaml @@ -1,4 +1,4 @@ -{{- if and (eq .Values.sumologic.logs.enabled true) (eq .Values.sumologic.logs.provider "fluentd") (eq .Values.fluentd.logs.enabled true) }} +{{- if eq (include "logs.enabled" .) "true" }} apiVersion: v1 kind: Service metadata: From 410f1ced10c6f5e69cf7c4d02b58d884102ef0ec Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Thu, 15 Jul 2021 13:58:13 +0200 Subject: [PATCH 04/37] feat(helm,logs): add logs.otelcol.enabled and logs.fluentd.enabled helpers Signed-off-by: Dominik Rosiek --- deploy/helm/sumologic/templates/_helpers.tpl | 36 +++++++++++++++++-- .../templates/logs/fluentd/configmap.yaml | 2 +- .../templates/logs/fluentd/statefulset.yaml | 2 +- .../templates/logs/otelcol/configmap.yaml | 2 +- .../templates/logs/otelcol/statefulset.yaml | 2 +- 5 files changed, 38 insertions(+), 6 deletions(-) diff --git a/deploy/helm/sumologic/templates/_helpers.tpl b/deploy/helm/sumologic/templates/_helpers.tpl index 263cfbe442..b5e99ef42e 100644 --- a/deploy/helm/sumologic/templates/_helpers.tpl +++ b/deploy/helm/sumologic/templates/_helpers.tpl @@ -1094,10 +1094,26 @@ Example Usage: */}} {{- define "logs.enabled" -}} {{- $enabled := false -}} -{{- if eq .Values.sumologic.logs.enabled true -}} -{{- if and (eq .Values.sumologic.logs.provider "fluentd") (eq .Values.fluentd.logs.enabled true) -}} +{{- if eq (include "logs.otelcol.enabled" .) "true" }} +{{- $enabled = true -}} +{{- end -}} +{{- if eq (include "logs.fluentd.enabled" .) "true" }} {{- $enabled = true -}} {{- end -}} +{{ $enabled }} +{{- end -}} + + +{{/* +Check if otelcol logs provider is enabled + +Example Usage: +{{- if eq (include "logs.otelcol.enabled" .) "true" }} + +*/}} +{{- define "logs.otelcol.enabled" -}} +{{- $enabled := false -}} +{{- if eq .Values.sumologic.logs.enabled true -}} {{- if and (eq .Values.sumologic.logs.provider "otelcol") (eq .Values.otelcol.logs.enabled true) -}} {{- $enabled = true -}} {{- end -}} @@ -1105,6 +1121,22 @@ Example Usage: {{ $enabled }} {{- end -}} +{{/* +Check if fluentd logs provider is enabled + +Example Usage: +{{- if eq (include "logs.fluentd.enabled" .) "true" }} + +*/}} +{{- define "logs.fluentd.enabled" -}} +{{- $enabled := false -}} +{{- if eq .Values.sumologic.logs.enabled true -}} +{{- if and (eq .Values.sumologic.logs.provider "fluentd") (eq .Values.fluentd.logs.enabled true) -}} +{{- $enabled = true -}} +{{- end -}} +{{- end -}} +{{ $enabled }} +{{- end -}} {{/* Add service labels diff --git a/deploy/helm/sumologic/templates/logs/fluentd/configmap.yaml b/deploy/helm/sumologic/templates/logs/fluentd/configmap.yaml index 20653a8162..a949806689 100644 --- a/deploy/helm/sumologic/templates/logs/fluentd/configmap.yaml +++ b/deploy/helm/sumologic/templates/logs/fluentd/configmap.yaml @@ -1,4 +1,4 @@ -{{- if and (eq .Values.sumologic.logs.enabled true) (eq .Values.sumologic.logs.provider "fluentd") (eq .Values.fluentd.logs.enabled true) }} +{{- if eq (include "logs.fluentd.enabled" .) "true" }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/deploy/helm/sumologic/templates/logs/fluentd/statefulset.yaml b/deploy/helm/sumologic/templates/logs/fluentd/statefulset.yaml index 2e718e4e36..5849d2e0d3 100644 --- a/deploy/helm/sumologic/templates/logs/fluentd/statefulset.yaml +++ b/deploy/helm/sumologic/templates/logs/fluentd/statefulset.yaml @@ -1,4 +1,4 @@ -{{- if and (eq .Values.sumologic.logs.enabled true) (eq .Values.sumologic.logs.provider "fluentd") (eq .Values.fluentd.logs.enabled true) }} +{{- if eq (include "logs.fluentd.enabled" .) "true" }} apiVersion: apps/v1 kind: StatefulSet metadata: diff --git a/deploy/helm/sumologic/templates/logs/otelcol/configmap.yaml b/deploy/helm/sumologic/templates/logs/otelcol/configmap.yaml index cacc706e5b..83f4ad8c1c 100644 --- a/deploy/helm/sumologic/templates/logs/otelcol/configmap.yaml +++ b/deploy/helm/sumologic/templates/logs/otelcol/configmap.yaml @@ -1,4 +1,4 @@ -{{- if and (eq .Values.sumologic.logs.enabled true) (eq .Values.sumologic.logs.provider "otelcol") (eq .Values.otelcol.logs.enabled true) }} +{{- if eq (include "logs.otelcol.enabled" .) "true" }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml b/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml index 59f956f28d..16555b75ba 100644 --- a/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml +++ b/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml @@ -1,4 +1,4 @@ -{{- if and (eq .Values.sumologic.logs.enabled true) (eq .Values.sumologic.logs.provider "otelcol") (eq .Values.otelcol.logs.enabled true) }} +{{- if eq (include "logs.otelcol.enabled" .) "true" }} apiVersion: apps/v1 kind: StatefulSet metadata: From c0fbb586f7297f0c6390da657bd270a3cbd8c811 Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Fri, 16 Jul 2021 07:43:45 +0200 Subject: [PATCH 05/37] feat(helm,logs,otelcol): add k8s metadata Signed-off-by: Dominik Rosiek --- deploy/helm/sumologic/values.yaml | 88 ++++++++++++++++++++++++++++++- 1 file changed, 87 insertions(+), 1 deletion(-) diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 758705fe12..2fe4c6f00f 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -2636,6 +2636,87 @@ otelcol: sumologic: log_format: json endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE} + processors: + attributes: + actions: + - action: extract + key: fluent.tag + pattern: ^containers\.var\.log\.pods\.(?P[^_]+)_(?P[^_]+)_(?P[a-f0-9\-]{36})\.(?P[^\._]+)\.(?P\d+)\.log$ + - action: delete + key: fluent.tag + - action: insert + key: k8s.pod.name + from_attribute: pod_name + - action: delete + key: pod_name + - action: insert + key: k8s.pod.uid + from_attribute: uid + - action: delete + key: uid + - action: insert + key: k8s.namespace.name + from_attribute: namespace + - action: delete + key: namespace + - action: insert + key: k8s.container.name + from_attribute: container_name + - action: delete + key: container_name + - action: insert + key: k8s.instance.id + from_attribute: run_id + - action: delete + key: run_id + groupbyattrs: + keys: + - fluent.tag + - k8s.container.name + - k8s.instance.id + - k8s.namespace.name + - k8s.pod.name + - k8s.pod.uid + batch: + send_batch_size: 10000 + timeout: 10s + resource: + attributes: + - key: agent + value: otelcol + action: insert + k8s_tagger: + # Has to be false to enrich metadata + passthrough: false + owner_lookup_enabled: true # To enable fetching additional metadata using `owner` relationship + extract: + metadata: + ## extract the following well-known metadata fields + - containerId + - containerName + - clusterName + - daemonSetName + - deploymentName + - hostName + - namespace + - nodeName + - podId + - podName + - replicaSetName + - serviceName + - statefulSetName + annotations: + - tag_name: "pod_annotations_%s" + key: "*" + namespace_labels: + - tag_name: "namespace_labels_%s" + key: "*" + labels: + - tag_name: "pod_labels_%s" + key: "*" + pod_association: + - from: resource_attribute + name: k8s.pod.uid service: extensions: - health_check @@ -2644,7 +2725,12 @@ otelcol: logs: receivers: - fluentforward - processors: [] + processors: + - attributes + - groupbyattrs + - batch + - k8s_tagger + - resource exporters: - sumologic deployment: From 501b33c665dc2ef48407566ecbb22dcdff5823d5 Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Mon, 19 Jul 2021 09:34:52 +0200 Subject: [PATCH 06/37] feat(helm,logs): add common objects for logs pipeline Signed-off-by: Dominik Rosiek --- deploy/helm/sumologic/templates/_helpers.tpl | 2 + .../sumologic/templates/logs/common/hpa.yaml | 31 +++++++++++++ .../sumologic/templates/logs/common/pdb.yaml | 13 ++++++ .../logs/common/service-headless.yaml | 31 +++++++++++++ .../templates/logs/common/service.yaml | 31 +++++++++++++ .../sumologic/templates/logs/fluentd/hpa.yaml | 2 +- .../sumologic/templates/logs/fluentd/pdb.yaml | 2 +- .../logs/fluentd/service-headless.yaml | 2 +- .../templates/logs/fluentd/service.yaml | 2 +- deploy/helm/sumologic/values.yaml | 45 +++++++++++++++++++ 10 files changed, 157 insertions(+), 4 deletions(-) create mode 100644 deploy/helm/sumologic/templates/logs/common/hpa.yaml create mode 100644 deploy/helm/sumologic/templates/logs/common/pdb.yaml create mode 100644 deploy/helm/sumologic/templates/logs/common/service-headless.yaml create mode 100644 deploy/helm/sumologic/templates/logs/common/service.yaml diff --git a/deploy/helm/sumologic/templates/_helpers.tpl b/deploy/helm/sumologic/templates/_helpers.tpl index b5e99ef42e..50de7815b4 100644 --- a/deploy/helm/sumologic/templates/_helpers.tpl +++ b/deploy/helm/sumologic/templates/_helpers.tpl @@ -1138,8 +1138,10 @@ Example Usage: {{ $enabled }} {{- end -}} + {{/* Add service labels + Example Usage: {{- if eq (include "service.labels" dict("Provider" "fluentd" "Values" .Values)) "true" }} diff --git a/deploy/helm/sumologic/templates/logs/common/hpa.yaml b/deploy/helm/sumologic/templates/logs/common/hpa.yaml new file mode 100644 index 0000000000..948ed8dc23 --- /dev/null +++ b/deploy/helm/sumologic/templates/logs/common/hpa.yaml @@ -0,0 +1,31 @@ +{{- if eq (include "logs.otelcol.enabled" .) "true" }} +apiVersion: autoscaling/v2beta2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ template "sumologic.metadata.name.logs.hpa" . }} + labels: + app: {{ template "sumologic.labels.app.logs.hpa" . }} + {{- include "sumologic.labels.common" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: StatefulSet + name: {{ template "sumologic.metadata.name.logs.statefulset" . }} + minReplicas: {{ .Values.logs.autoscaling.minReplicas }} + maxReplicas: {{ .Values.logs.autoscaling.maxReplicas }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.logs.autoscaling.targetCPUUtilizationPercentage }} +{{- if .Values.logs.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.logs.autoscaling.targetMemoryUtilizationPercentage }} +{{- end -}} +{{- end -}} diff --git a/deploy/helm/sumologic/templates/logs/common/pdb.yaml b/deploy/helm/sumologic/templates/logs/common/pdb.yaml new file mode 100644 index 0000000000..ebd247e334 --- /dev/null +++ b/deploy/helm/sumologic/templates/logs/common/pdb.yaml @@ -0,0 +1,13 @@ +{{- if eq (include "logs.otelcol.enabled" .) "true" }} +{{- if .Values.logs.podDisruptionBudget -}} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: {{ template "sumologic.metadata.name.logs.pdb" . }} +spec: + selector: + matchLabels: + app: {{ template "sumologic.labels.app.logs.statefulset" . }} +{{ toYaml .Values.logs.podDisruptionBudget | indent 2 }} +{{- end -}} +{{- end -}} diff --git a/deploy/helm/sumologic/templates/logs/common/service-headless.yaml b/deploy/helm/sumologic/templates/logs/common/service-headless.yaml new file mode 100644 index 0000000000..2c661c28c3 --- /dev/null +++ b/deploy/helm/sumologic/templates/logs/common/service-headless.yaml @@ -0,0 +1,31 @@ +{{- if eq (include "logs.otelcol.enabled" .) "true" }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "sumologic.metadata.name.logs.service-headless" . }} + labels: + app: {{ template "sumologic.labels.app.logs.service-headless" . }} + {{- include "sumologic.labels.logs" . | nindent 4 }} +{{- if (include "service.labels" (dict "Provider" .Values.sumologic.logs.provider "Values" .Values )) }} +{{ include "service.labels" (dict "Provider" .Values.sumologic.logs.provider "Values" .Values ) | indent 4 }} +{{- end }} +spec: + selector: + app: {{ template "sumologic.labels.app.logs.pod" . }} + clusterIP: None + ports: + {{- if .Values.sumologic.traces.enabled }} + - name: zipkin-write + port: 9411 + targetPort: 9411 + protocol: TCP + {{- end }} + - name: fluent-bit + port: 24321 + targetPort: 24321 + protocol: TCP + - name: metrics + port: 24231 + targetPort: 24231 + protocol: TCP +{{- end }} diff --git a/deploy/helm/sumologic/templates/logs/common/service.yaml b/deploy/helm/sumologic/templates/logs/common/service.yaml new file mode 100644 index 0000000000..0b467e56aa --- /dev/null +++ b/deploy/helm/sumologic/templates/logs/common/service.yaml @@ -0,0 +1,31 @@ +{{- if eq (include "logs.otelcol.enabled" .) "true" }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "sumologic.metadata.name.logs.service" . }} + labels: + app: {{ template "sumologic.labels.app.logs.service" . }} + {{- include "sumologic.labels.scrape.logs" . | nindent 4 }} + {{- include "sumologic.labels.common" . | nindent 4 }} +{{- if (include "service.labels" (dict "Provider" .Values.sumologic.logs.provider "Values" .Values )) }} +{{ include "service.labels" (dict "Provider" .Values.sumologic.logs.provider "Values" .Values ) | indent 4 }} +{{- end }} +spec: + selector: + app: {{ template "sumologic.labels.app.logs.pod" . }} + ports: + {{- if .Values.sumologic.traces.enabled }} + - name: zipkin-write + port: 9411 + targetPort: 9411 + protocol: TCP + {{- end }} + - name: fluent-bit + port: 24321 + targetPort: 24321 + protocol: TCP + - name: metrics + port: 24231 + targetPort: 24231 + protocol: TCP +{{- end }} diff --git a/deploy/helm/sumologic/templates/logs/fluentd/hpa.yaml b/deploy/helm/sumologic/templates/logs/fluentd/hpa.yaml index 883eb4cbc6..ddacc3b09f 100644 --- a/deploy/helm/sumologic/templates/logs/fluentd/hpa.yaml +++ b/deploy/helm/sumologic/templates/logs/fluentd/hpa.yaml @@ -1,4 +1,4 @@ -{{- if eq (include "logs.enabled" .) "true" }} +{{- if eq (include "logs.fluentd.enabled" .) "true" }} apiVersion: autoscaling/v2beta2 kind: HorizontalPodAutoscaler metadata: diff --git a/deploy/helm/sumologic/templates/logs/fluentd/pdb.yaml b/deploy/helm/sumologic/templates/logs/fluentd/pdb.yaml index 29771c5b69..71c52b4006 100644 --- a/deploy/helm/sumologic/templates/logs/fluentd/pdb.yaml +++ b/deploy/helm/sumologic/templates/logs/fluentd/pdb.yaml @@ -1,4 +1,4 @@ -{{- if eq (include "logs.enabled" .) "true" }} +{{- if eq (include "logs.fluentd.enabled" .) "true" }} {{- if .Values.fluentd.logs.podDisruptionBudget -}} apiVersion: policy/v1beta1 kind: PodDisruptionBudget diff --git a/deploy/helm/sumologic/templates/logs/fluentd/service-headless.yaml b/deploy/helm/sumologic/templates/logs/fluentd/service-headless.yaml index 84b1a81361..a9515231fd 100644 --- a/deploy/helm/sumologic/templates/logs/fluentd/service-headless.yaml +++ b/deploy/helm/sumologic/templates/logs/fluentd/service-headless.yaml @@ -1,4 +1,4 @@ -{{- if eq (include "logs.enabled" .) "true" }} +{{- if eq (include "logs.fluentd.enabled" .) "true" }} apiVersion: v1 kind: Service metadata: diff --git a/deploy/helm/sumologic/templates/logs/fluentd/service.yaml b/deploy/helm/sumologic/templates/logs/fluentd/service.yaml index b79ea8fb15..8d7bd0672f 100644 --- a/deploy/helm/sumologic/templates/logs/fluentd/service.yaml +++ b/deploy/helm/sumologic/templates/logs/fluentd/service.yaml @@ -1,4 +1,4 @@ -{{- if eq (include "logs.enabled" .) "true" }} +{{- if eq (include "logs.fluentd.enabled" .) "true" }} apiVersion: v1 kind: Service metadata: diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 2fe4c6f00f..15ec555b18 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -2937,6 +2937,51 @@ otelcol: ## and uncomment next one # exporters: [zipkin] +logs: + statefulset: + nodeSelector: {} + tolerations: [] + affinity: {} + ## Acceptable values for podAntiAffinity: + ## soft: specifies preferences that the scheduler will try to enforce but will not guarantee (Default) + ## hard: specifies rules that must be met for a pod to be scheduled onto a node + podAntiAffinity: "soft" + replicaCount: 3 + resources: + limits: + memory: 1Gi + cpu: 1000m + requests: + memory: 768Mi + cpu: 500m + ## Option to define priorityClassName to assign a priority class to pods. + priorityClassName: + + ## Add custom labels only to logs fluentd sts pods + podLabels: {} + ## Add custom annotations only to logs fluentd sts pods + podAnnotations: {} + + ## Set securityContext for containers running in pods in logs statefulset. + containers: + fluentd: + securityContext: {} + + ## Option to turn autoscaling on for fluentd and specify params for HPA. + ## Autoscaling needs metrics-server to access cpu metrics. + autoscaling: + enabled: false + minReplicas: 3 + maxReplicas: 10 + targetCPUUtilizationPercentage: 50 + # targetMemoryUtilizationPercentage: 50 + + ## Option to specify PodDisrutionBudgets + ## You can specify only one of maxUnavailable and minAvailable in a single PodDisruptionBudget + podDisruptionBudget: + minAvailable: 2 + ## To use maxUnavailable, set minAvailable to null and uncomment the below: + # maxUnavailable: 1 ## Configure telegraf-operator ## ref: https://github.com/influxdata/helm-charts/blob/master/charts/telegraf-operator/values.yaml telegraf-operator: From fa55e27a2d2f21a4948cf8b8bd1ea4567eaf54da Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Mon, 19 Jul 2021 12:05:22 +0200 Subject: [PATCH 07/37] feat(helm,logs): add source headers to otelcol logs Signed-off-by: Dominik Rosiek --- deploy/helm/sumologic/values.yaml | 46 +++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 15ec555b18..ef4487c687 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -2636,6 +2636,28 @@ otelcol: sumologic: log_format: json endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE} + ## Set the _sourceName metadata field in Sumo Logic. + ## sourceName: "%{namespace}.%{pod}.%{container}" + ## Set the _sourceCategory metadata field in Sumo Logic. + ## sourceCategory: "%{namespace}/%{pod_name}" + source_name: "%{k8s.namespace.name}.%{k8s.pod.name}.%{k8s.container.name}" + ## ToDo: Replace %{k8s.pod.name}" with sanitized pod name + ## ToDo: Add `-` replacement configuration, by default with `/` + source_category: "%{k8s.namespace.name}/%{k8s.pod.name}" + source_host: '%{k8s.pod.hostname}' + metadata_attributes: + - k8s.* + - pod_.* + - namespace_.* + + - Deployment + - Namespace + - container + - host + - node + - pod + ## ToDo: Remove before merge + - agent processors: attributes: actions: @@ -2677,9 +2699,26 @@ otelcol: - k8s.namespace.name - k8s.pod.name - k8s.pod.uid + + ## The memory_limiter processor is used to prevent out of memory situations on the collector. + memory_limiter: + ## check_interval is the time between measurements of memory usage for the + ## purposes of avoiding going over the limits. Defaults to zero, so no + ## checks will be performed. Values below 1 second are not recommended since + ## it can result in unnecessary CPU consumption. + check_interval: 5s + + ## Maximum amount of memory, in MiB, targeted to be allocated by the process heap. + ## Note that typically the total memory usage of process will be about 50MiB higher + ## than this value. + limit_mib: 1900 + + ## The batch processor accepts spans and places them into batches grouped by node and resource batch: - send_batch_size: 10000 - timeout: 10s + ## Number of spans after which a batch will be sent regardless of time + send_batch_size: 256 + ## Time duration after which a batch will be sent regardless of size + timeout: 5s resource: attributes: - key: agent @@ -2726,11 +2765,12 @@ otelcol: receivers: - fluentforward processors: + - memory_limiter - attributes - groupbyattrs - - batch - k8s_tagger - resource + - batch exporters: - sumologic deployment: From 4d9fd4faa9d1292541514dedb25124841e8a9a1c Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Mon, 19 Jul 2021 13:43:25 +0200 Subject: [PATCH 08/37] refactor(yamllint): update values.yaml Signed-off-by: Dominik Rosiek --- deploy/helm/sumologic/values.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index ef4487c687..d9ca351914 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -2721,13 +2721,13 @@ otelcol: timeout: 5s resource: attributes: - - key: agent - value: otelcol - action: insert + - key: agent + value: otelcol + action: insert k8s_tagger: - # Has to be false to enrich metadata + ## Has to be false to enrich metadata passthrough: false - owner_lookup_enabled: true # To enable fetching additional metadata using `owner` relationship + owner_lookup_enabled: true # To enable fetching additional metadata using `owner` relationship extract: metadata: ## extract the following well-known metadata fields @@ -2754,8 +2754,8 @@ otelcol: - tag_name: "pod_labels_%s" key: "*" pod_association: - - from: resource_attribute - name: k8s.pod.uid + - from: resource_attribute + name: k8s.pod.uid service: extensions: - health_check From 1cb8e5278e28ecf5c43a165c3700d32a2ffa6541 Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Mon, 19 Jul 2021 14:06:50 +0200 Subject: [PATCH 09/37] refactor(helm,otelcol): converts fluentd-specific values to common for otelcol Signed-off-by: Dominik Rosiek --- .../templates/logs/otelcol/statefulset.yaml | 79 +++++++++++-------- deploy/helm/sumologic/values.yaml | 38 ++++++++- 2 files changed, 83 insertions(+), 34 deletions(-) diff --git a/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml b/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml index 16555b75ba..dc077d1d94 100644 --- a/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml +++ b/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml @@ -12,8 +12,8 @@ spec: app: {{ template "sumologic.labels.app.logs.pod" . }} serviceName: {{ template "sumologic.metadata.name.logs.service-headless" . }} podManagementPolicy: "Parallel" -{{- if not .Values.fluentd.logs.autoscaling.enabled }} - replicas: {{ .Values.fluentd.logs.statefulset.replicaCount }} +{{- if not .Values.logs.autoscaling.enabled }} + replicas: {{ .Values.logs.statefulset.replicaCount }} {{- end }} template: metadata: @@ -22,11 +22,11 @@ spec: {{- if .Values.sumologic.podAnnotations }} {{ toYaml .Values.sumologic.podAnnotations | indent 8 }} {{- end }} -{{- if .Values.fluentd.podAnnotations }} -{{ toYaml .Values.fluentd.podAnnotations | indent 8 }} +{{- if .Values.otelcol.podAnnotations }} +{{ toYaml .Values.otelcol.podAnnotations | indent 8 }} {{- end }} -{{- if .Values.fluentd.logs.statefulset.podAnnotations }} -{{ toYaml .Values.fluentd.logs.statefulset.podAnnotations | indent 8 }} +{{- if .Values.logs.statefulset.podAnnotations }} +{{ toYaml .Values.logs.statefulset.podAnnotations | indent 8 }} {{- end }} labels: app: {{ template "sumologic.labels.app.logs.pod" . }} @@ -34,28 +34,28 @@ spec: {{- if .Values.sumologic.podLabels }} {{ toYaml .Values.sumologic.podLabels | indent 8 }} {{- end }} -{{- if .Values.fluentd.podLabels }} -{{ toYaml .Values.fluentd.podLabels | indent 8 }} +{{- if .Values.otelcol.podLabels }} +{{ toYaml .Values.otelcol.podLabels | indent 8 }} {{- end }} -{{- if .Values.fluentd.logs.statefulset.podLabels }} -{{ toYaml .Values.fluentd.logs.statefulset.podLabels | indent 8 }} +{{- if .Values.logs.statefulset.podLabels }} +{{ toYaml .Values.logs.statefulset.podLabels | indent 8 }} {{- end }} spec: serviceAccountName: {{ template "sumologic.metadata.name.roles.serviceaccount" . }} -{{- if .Values.fluentd.logs.statefulset.nodeSelector }} +{{- if .Values.logs.statefulset.nodeSelector }} nodeSelector: -{{ toYaml .Values.fluentd.logs.statefulset.nodeSelector | indent 8 }} +{{ toYaml .Values.logs.statefulset.nodeSelector | indent 8 }} {{- end }} -{{- if .Values.fluentd.logs.statefulset.tolerations }} +{{- if .Values.logs.statefulset.tolerations }} tolerations: -{{ toYaml .Values.fluentd.logs.statefulset.tolerations | indent 8 }} +{{ toYaml .Values.logs.statefulset.tolerations | indent 8 }} {{- end }} -{{- if or .Values.fluentd.logs.statefulset.affinity .Values.fluentd.logs.statefulset.podAntiAffinity }} +{{- if or .Values.logs.statefulset.affinity .Values.fluentd.logs.statefulset.podAntiAffinity }} affinity: -{{- if .Values.fluentd.logs.statefulset.affinity }} -{{ toYaml .Values.fluentd.logs.statefulset.affinity | indent 8 }} +{{- if .Values.logs.statefulset.affinity }} +{{ toYaml .Values.logs.statefulset.affinity | indent 8 }} {{- end }} -{{- if eq .Values.fluentd.logs.statefulset.podAntiAffinity "hard"}} +{{- if eq .Values.logs.statefulset.podAntiAffinity "hard"}} podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: @@ -70,7 +70,7 @@ spec: values: - prometheus-operator-prometheus topologyKey: "kubernetes.io/hostname" -{{else if eq .Values.fluentd.logs.statefulset.podAntiAffinity "soft"}} +{{else if eq .Values.logs.statefulset.podAntiAffinity "soft"}} podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 @@ -93,23 +93,23 @@ spec: - name: config-volume configMap: name: {{ template "sumologic.metadata.name.logs.configmap" . }} -{{- if .Values.fluentd.logs.extraVolumes }} -{{ toYaml .Values.fluentd.logs.extraVolumes | indent 6 }} +{{- if .Values.logs.extraVolumes }} +{{ toYaml .Values.logs.extraVolumes | indent 6 }} {{- end }} securityContext: - {{- toYaml .Values.fluentd.securityContext | nindent 8 }} - {{- if .Values.fluentd.logs.statefulset.priorityClassName }} - priorityClassName: {{ .Values.fluentd.logs.statefulset.priorityClassName | quote }} + {{- toYaml .Values.otelcol.securityContext | nindent 8 }} + {{- if .Values.logs.statefulset.priorityClassName }} + priorityClassName: {{ .Values.logs.statefulset.priorityClassName | quote }} {{- end }} containers: - name: otelcol image: {{ .Values.otelcol.statefulset.image.repository }}:{{ .Values.otelcol.statefulset.image.tag }} imagePullPolicy: {{ .Values.otelcol.statefulset.image.pullPolicy }} resources: - {{- toYaml .Values.fluentd.logs.statefulset.resources | nindent 10 }} - {{- if .Values.fluentd.logs.statefulset.containers.fluentd.securityContext }} + {{- toYaml .Values.logs.statefulset.resources | nindent 10 }} + {{- if .Values.logs.statefulset.containers.otelcol.securityContext }} securityContext: - {{- toYaml .Values.fluentd.otelcol.statefulset.containers.otelcol.securityContext | nindent 10 }} + {{- toYaml .Values.logs.statefulset.containers.otelcol.securityContext | nindent 10 }} {{- end }} ports: {{- if .Values.sumologic.traces.enabled }} @@ -132,8 +132,8 @@ spec: - name: config-volume mountPath: /etc/otel/config.yaml subPath: config.yaml -{{- if .Values.fluentd.logs.extraVolumeMounts }} -{{ toYaml .Values.fluentd.logs.extraVolumeMounts | indent 8 }} +{{- if .Values.logs.extraVolumeMounts }} +{{ toYaml .Values.logs.extraVolumeMounts | indent 8 }} {{- end }} env: {{- $ctx := .Values -}} @@ -141,9 +141,22 @@ spec: {{- if .Values.sumologic.traces.enabled }} {{ include "kubernetes.sources.envs" (dict "Context" $ctx "Type" "traces")}} {{- end }} - {{- if .Values.fluentd.logs.extraEnvVars }} -{{ toYaml .Values.fluentd.logs.extraEnvVars | nindent 8 }} + {{- if .Values.logs.extraEnvVars }} +{{ toYaml .Values.logs.extraEnvVars | nindent 8 }} {{- end }} - - name: ADDITIONAL_PLUGINS - value: {{ join " " .Values.fluentd.additionalPlugins | quote }} +{{- if .Values.logs.persistence.enabled }} + volumeClaimTemplates: + - metadata: + name: buffer +{{- if .Values.logs.pvcLabels }} + labels: +{{ toYaml .Values.logs.pvcLabels | indent 8 }} +{{- end }} + spec: + accessModes: [{{ .Values.logs.persistence.accessMode }}] + storageClassName: {{ .Values.logs.persistence.storageClass }} + resources: + requests: + storage: {{ .Values.logs.persistence.size }} +{{- end }} {{- end }} diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index d9ca351914..98b1eab609 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -3004,9 +3004,27 @@ logs: ## Set securityContext for containers running in pods in logs statefulset. containers: - fluentd: + otelcol: securityContext: {} + ## Extra Environment Values - allows yaml definitions + # extraEnvVars: + # - name: VALUE_FROM_SECRET + # valueFrom: + # secretKeyRef: + # name: secret_name + # key: secret_key + + # extraVolumes: + # - name: es-certs + # secret: + # defaultMode: 420 + # secretName: es-certs + # extraVolumeMounts: + # - name: es-certs + # mountPath: /certs + # readOnly: true + ## Option to turn autoscaling on for fluentd and specify params for HPA. ## Autoscaling needs metrics-server to access cpu metrics. autoscaling: @@ -3022,6 +3040,24 @@ logs: minAvailable: 2 ## To use maxUnavailable, set minAvailable to null and uncomment the below: # maxUnavailable: 1 + + ## Persist data to a persistent volume; When enabled, fluentd uses the file buffer instead of memory buffer. + persistence: + ## After changing this value please follow steps described in: + ## https://github.com/SumoLogic/sumologic-kubernetes-collection/blob/main/deploy/docs/FluentdPersistence.md + enabled: true + + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, Azure & OpenStack) + ## + # storageClass: "-" + # annotations: {} + accessMode: ReadWriteOnce + size: 10Gi + ## Configure telegraf-operator ## ref: https://github.com/influxdata/helm-charts/blob/master/charts/telegraf-operator/values.yaml telegraf-operator: From 1dff83135cab6bb70c90f0518f09bd4ade42a13a Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Mon, 19 Jul 2021 14:29:08 +0200 Subject: [PATCH 10/37] refactor(helm,logs,otelcol): add persistance todo Signed-off-by: Dominik Rosiek --- .../templates/logs/otelcol/statefulset.yaml | 15 --------------- deploy/helm/sumologic/values.yaml | 1 + 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml b/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml index dc077d1d94..5f6fccd6bd 100644 --- a/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml +++ b/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml @@ -144,19 +144,4 @@ spec: {{- if .Values.logs.extraEnvVars }} {{ toYaml .Values.logs.extraEnvVars | nindent 8 }} {{- end }} -{{- if .Values.logs.persistence.enabled }} - volumeClaimTemplates: - - metadata: - name: buffer -{{- if .Values.logs.pvcLabels }} - labels: -{{ toYaml .Values.logs.pvcLabels | indent 8 }} -{{- end }} - spec: - accessModes: [{{ .Values.logs.persistence.accessMode }}] - storageClassName: {{ .Values.logs.persistence.storageClass }} - resources: - requests: - storage: {{ .Values.logs.persistence.size }} -{{- end }} {{- end }} diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 98b1eab609..16afb73a22 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -3042,6 +3042,7 @@ logs: # maxUnavailable: 1 ## Persist data to a persistent volume; When enabled, fluentd uses the file buffer instead of memory buffer. + ## ToDo: implement on otelcol side persistence: ## After changing this value please follow steps described in: ## https://github.com/SumoLogic/sumologic-kubernetes-collection/blob/main/deploy/docs/FluentdPersistence.md From 2fe73f40c7b9b8775b7aecc5640c5bdafceec5bb Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Mon, 19 Jul 2021 14:30:02 +0200 Subject: [PATCH 11/37] refactor(helm,logs,otelcol): remove traves envs from logs sts Signed-off-by: Dominik Rosiek --- deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml b/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml index 5f6fccd6bd..acd44d7749 100644 --- a/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml +++ b/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml @@ -138,9 +138,6 @@ spec: env: {{- $ctx := .Values -}} {{ include "kubernetes.sources.envs" (dict "Context" $ctx "Type" "logs")}} -{{- if .Values.sumologic.traces.enabled }} -{{ include "kubernetes.sources.envs" (dict "Context" $ctx "Type" "traces")}} -{{- end }} {{- if .Values.logs.extraEnvVars }} {{ toYaml .Values.logs.extraEnvVars | nindent 8 }} {{- end }} From 30cdd3c68cee89a79f0ba0d29cd775c03876153f Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Mon, 19 Jul 2021 14:31:52 +0200 Subject: [PATCH 12/37] refactor(helm,otelcol,logs): remove agent attribute Signed-off-by: Dominik Rosiek --- deploy/helm/sumologic/values.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 16afb73a22..e5f0ccd443 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -2656,8 +2656,6 @@ otelcol: - host - node - pod - ## ToDo: Remove before merge - - agent processors: attributes: actions: @@ -2719,11 +2717,6 @@ otelcol: send_batch_size: 256 ## Time duration after which a batch will be sent regardless of size timeout: 5s - resource: - attributes: - - key: agent - value: otelcol - action: insert k8s_tagger: ## Has to be false to enrich metadata passthrough: false @@ -2769,7 +2762,6 @@ otelcol: - attributes - groupbyattrs - k8s_tagger - - resource - batch exporters: - sumologic From be26e004f02d44d81e9dc16270ee16e7f65ad56f Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Mon, 19 Jul 2021 14:32:44 +0200 Subject: [PATCH 13/37] refactor(helm,otelcol,logs): clean up config Signed-off-by: Dominik Rosiek --- deploy/helm/sumologic/values.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index e5f0ccd443..6d58ee7db9 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -2691,7 +2691,6 @@ otelcol: key: run_id groupbyattrs: keys: - - fluent.tag - k8s.container.name - k8s.instance.id - k8s.namespace.name From 691ca801a3726633c1cea3f6163c21b57c6ef26d Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Mon, 19 Jul 2021 14:35:59 +0200 Subject: [PATCH 14/37] refactor,docs: update otelcol related comments Signed-off-by: Dominik Rosiek --- deploy/helm/sumologic/values.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 6d58ee7db9..4c9c81b279 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -2968,6 +2968,9 @@ otelcol: ## and uncomment next one # exporters: [zipkin] +## Configure logs pipeline. +## This section affects only otelcol provider. +## In the future it will be used by fluentd as well. logs: statefulset: nodeSelector: {} @@ -2988,9 +2991,9 @@ logs: ## Option to define priorityClassName to assign a priority class to pods. priorityClassName: - ## Add custom labels only to logs fluentd sts pods + ## Add custom labels only to logs sts pods podLabels: {} - ## Add custom annotations only to logs fluentd sts pods + ## Add custom annotations only to logs sts pods podAnnotations: {} ## Set securityContext for containers running in pods in logs statefulset. @@ -3016,7 +3019,7 @@ logs: # mountPath: /certs # readOnly: true - ## Option to turn autoscaling on for fluentd and specify params for HPA. + ## Option to turn autoscaling on for logs and specify params for HPA. ## Autoscaling needs metrics-server to access cpu metrics. autoscaling: enabled: false From a747cb20399acea1640c8253a8c446081e52d6a1 Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Tue, 20 Jul 2021 08:16:56 +0200 Subject: [PATCH 15/37] feat(helm,logs): add check for provider values Signed-off-by: Dominik Rosiek --- deploy/helm/sumologic/templates/checks.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/deploy/helm/sumologic/templates/checks.txt b/deploy/helm/sumologic/templates/checks.txt index 6388607570..a97bb710d9 100644 --- a/deploy/helm/sumologic/templates/checks.txt +++ b/deploy/helm/sumologic/templates/checks.txt @@ -4,7 +4,13 @@ {{- fail "\nAt least k8s 1.16 is required. Please update your k8s version or set sumologic.setup.force to true" -}} {{- end -}} {{- end -}} + {{/* Check if metrics provider is set to either fluentd or otelcol */}} {{- if not (or (eq .Values.sumologic.metrics.metadata.provider "fluentd") (eq .Values.sumologic.metrics.metadata.provider "otelcol")) -}} {{- fail "\nsumologic.metrics.provider should be set to either fluentd or otelcol" -}} {{- end -}} + +{{/* Check if logs provider is set to either fluentd or otelcol */}} +{{- if not (or (eq .Values.sumologic.logs.provider "fluentd") (eq .Values.sumologic.logs.provider "otelcol")) -}} +{{- fail "\nsumologic.logs.provider should be set to either fluentd or otelcol" -}} +{{- end -}} From 0ca797cd6b40aa830993892cdd820368d202c8a2 Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Fri, 6 Aug 2021 14:37:28 +0200 Subject: [PATCH 16/37] feat(helm,logs,otelcol): add source processor Signed-off-by: Dominik Rosiek --- deploy/helm/sumologic/values.yaml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 4c9c81b279..00e617f3ca 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -2747,7 +2747,24 @@ otelcol: key: "*" pod_association: - from: resource_attribute - name: k8s.pod.uid + name: k8s.pod.uid ## Source processor adds Sumo Logic related metadata + source: + collector: '{{ .Values.sumologic.collectorName | default .Values.sumologic.clusterName | quote }}' + source_name: '{{ .Values.fluentd.logs.containers.sourceName | quote }}' + source_category: '{{ .Values.fluentd.logs.containers.sourceCategory | quote }}' + source_category_prefix: '{{ .Values.fluentd.logs.containers.sourceCategoryPrefix | quote }}' + source_category_replace_dash: '{{ .Values.fluentd.logs.containers.sourceCategoryReplaceDash | quote }}' + exclude_namespace_regex: '{{ include "fluentd.excludeNamespaces" . }}' + exclude_pod_regex: '{{ .Values.fluentd.logs.containers.excludePodRegex | quote }}' + exclude_container_regex: '{{ .Values.fluentd.logs.containers.excludeContainerRegex | quote }}' + exclude_host_regex: '{{ .Values.fluentd.logs.containers.excludeHostRegex | quote }}' + annotation_prefix: "k8s.pod.annotation." + pod_template_hash_key: "k8s.pod.label.pod-template-hash" + pod_name_key: "k8s.pod.pod_name" + namespace_key: "k8s.namespace.name" + pod_key: "k8s.pod.name" + container_key: "k8s.container.name" + source_host_key: "k8s.pod.hostname" service: extensions: - health_check @@ -2761,6 +2778,7 @@ otelcol: - attributes - groupbyattrs - k8s_tagger + - source - batch exporters: - sumologic From 63d0d2e2fdba493eb8858a37a3767ff515d90c2d Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Mon, 9 Aug 2021 14:25:01 +0200 Subject: [PATCH 17/37] feat(helm): add experimental note to provider Signed-off-by: Dominik Rosiek --- deploy/helm/sumologic/values.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 00e617f3ca..dc7597cba5 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -184,7 +184,8 @@ sumologic: ## Set the enabled flag to false for disabling logs ingestion altogether. logs: enabled: true - ## Set provider service (either fluentd or otelcol) + ## Set provider service (either fluentd or otelcol). + ## Otelcol is experimental and may behave in unexpected way. provider: fluentd ## Fields to be created at Sumo Logic to ensure logs are tagged with From 25dac5f152a2d66ee69cc30101e55110cf5ab856 Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Mon, 16 Aug 2021 09:50:33 +0200 Subject: [PATCH 18/37] fix(helm): revert autoscalling option for fluent hpa Signed-off-by: Dominik Rosiek --- deploy/helm/sumologic/templates/logs/fluentd/hpa.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/helm/sumologic/templates/logs/fluentd/hpa.yaml b/deploy/helm/sumologic/templates/logs/fluentd/hpa.yaml index ddacc3b09f..4e3213fe19 100644 --- a/deploy/helm/sumologic/templates/logs/fluentd/hpa.yaml +++ b/deploy/helm/sumologic/templates/logs/fluentd/hpa.yaml @@ -1,4 +1,4 @@ -{{- if eq (include "logs.fluentd.enabled" .) "true" }} +{{- if and (eq (include "logs.fluentd.enabled" .) "true") (.Values.fluentd.logs.autoscaling.enabled) }} apiVersion: autoscaling/v2beta2 kind: HorizontalPodAutoscaler metadata: From ac2127db04bd6b9b6a78999bfe5b8be50f375e21 Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Mon, 16 Aug 2021 09:50:56 +0200 Subject: [PATCH 19/37] fix(helm): use autoscalling.enabled option for common hpa Signed-off-by: Dominik Rosiek --- deploy/helm/sumologic/templates/logs/common/hpa.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/helm/sumologic/templates/logs/common/hpa.yaml b/deploy/helm/sumologic/templates/logs/common/hpa.yaml index 948ed8dc23..07d7ca392b 100644 --- a/deploy/helm/sumologic/templates/logs/common/hpa.yaml +++ b/deploy/helm/sumologic/templates/logs/common/hpa.yaml @@ -1,4 +1,4 @@ -{{- if eq (include "logs.otelcol.enabled" .) "true" }} +{{- if and (eq (include "logs.otelcol.enabled" .) "true") (.Values.logs.autoscaling.enabled) }} apiVersion: autoscaling/v2beta2 kind: HorizontalPodAutoscaler metadata: From 6f958bccf814b177d93d76a92b5300d4d13b9016 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Wed, 8 Sep 2021 10:03:56 +0200 Subject: [PATCH 20/37] fix(helm): fix extracting k8s metadata from fluent.tag --- deploy/helm/sumologic/values.yaml | 32 +++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index dc7597cba5..e905cee7e8 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -2662,36 +2662,36 @@ otelcol: actions: - action: extract key: fluent.tag - pattern: ^containers\.var\.log\.pods\.(?P[^_]+)_(?P[^_]+)_(?P[a-f0-9\-]{36})\.(?P[^\._]+)\.(?P\d+)\.log$ + pattern: ^containers\.var\.log\.containers\.(?P[^_]+)_(?P[^_]+)_(?P.+)-(?P[a-f0-9]{64})\.log$ - action: delete key: fluent.tag + - action: insert - key: k8s.pod.name - from_attribute: pod_name + key: k8s.container.id + from_attribute: container_id - action: delete - key: pod_name + key: container_id + - action: insert - key: k8s.pod.uid - from_attribute: uid + key: k8s.pod.name + from_attribute: k8s_pod_name - action: delete - key: uid + key: k8s_pod_name + - action: insert key: k8s.namespace.name - from_attribute: namespace + from_attribute: k8s_namespace - action: delete - key: namespace + key: k8s_namespace + - action: insert key: k8s.container.name - from_attribute: container_name - - action: delete - key: container_name - - action: insert - key: k8s.instance.id - from_attribute: run_id + from_attribute: k8s_container_name - action: delete - key: run_id + key: k8s_container_name groupbyattrs: keys: + - k8s.container.id - k8s.container.name - k8s.instance.id - k8s.namespace.name From 96c1ddd0a76d9f6b57cfe89fc06df04d6d44a6be Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Wed, 8 Sep 2021 11:41:44 +0200 Subject: [PATCH 21/37] fix: remove `sumologic.open_endpoint` property from values.yaml It is not going to be used, as the OtelCol metadata provider will be using the same `sumologic.endpoint` property as Fluentd. --- deploy/helm/sumologic/values.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index e905cee7e8..a00ef5233b 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -27,7 +27,6 @@ sumologic: ## Sumo API endpoint; Leave blank for automatic endpoint discovery and redirection ## ref: https://help.sumologic.com/APIs/General-API-Information/Sumo-Logic-Endpoints-and-Firewall-Security endpoint: "" - open_endpoint: "" ## proxy urls httpProxy: "" From 20c366a26d2fb80b1ea7631c1156c781a17c8ca1 Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Wed, 8 Sep 2021 11:44:37 +0200 Subject: [PATCH 22/37] cleanup: remove commented-out sumologic extension It is not going to be used. OtelCol will be using the same endpoint as Fluentd is using now, as opposed to the new open endpoint, so the extension is not needed. --- deploy/helm/sumologic/values.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index a00ef5233b..2a866f65aa 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -2627,11 +2627,6 @@ otelcol: endpoint: 0.0.0.0:24321 extensions: health_check: {} - # sumologic: - # access_id: '{{ .Values.sumologic.accessId }}' - # access_key: '{{ .Values.sumologic.accessKey }}' - # collector_name: otc-{{ .Values.sumologic.collectorName | default .Values.sumologic.clusterName}} - # api_base_url: '{{ .Values.sumologic.open_endpoint }}' exporters: sumologic: log_format: json From 14a8943dd314eb7cc9a495abcfcd0123ce3ee03f Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Wed, 8 Sep 2021 11:46:45 +0200 Subject: [PATCH 23/37] cleanup: remove unused comments --- deploy/helm/sumologic/values.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 2a866f65aa..9acd4cd2f1 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -2631,10 +2631,6 @@ otelcol: sumologic: log_format: json endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE} - ## Set the _sourceName metadata field in Sumo Logic. - ## sourceName: "%{namespace}.%{pod}.%{container}" - ## Set the _sourceCategory metadata field in Sumo Logic. - ## sourceCategory: "%{namespace}/%{pod_name}" source_name: "%{k8s.namespace.name}.%{k8s.pod.name}.%{k8s.container.name}" ## ToDo: Replace %{k8s.pod.name}" with sanitized pod name ## ToDo: Add `-` replacement configuration, by default with `/` From 352c3db74af07cb61e3e43d904abf39924b6f0a5 Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Wed, 8 Sep 2021 11:47:40 +0200 Subject: [PATCH 24/37] fix: use sanitized pod name in _sourceCategory for logs --- deploy/helm/sumologic/values.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 9acd4cd2f1..fb35beade6 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -2632,9 +2632,8 @@ otelcol: log_format: json endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE} source_name: "%{k8s.namespace.name}.%{k8s.pod.name}.%{k8s.container.name}" - ## ToDo: Replace %{k8s.pod.name}" with sanitized pod name ## ToDo: Add `-` replacement configuration, by default with `/` - source_category: "%{k8s.namespace.name}/%{k8s.pod.name}" + source_category: "%{k8s.namespace.name}/%{k8s.pod.pod_name}" source_host: '%{k8s.pod.hostname}' metadata_attributes: - k8s.* From cb4814ffb85e345daf41b219d390ef3b39d6b358 Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Wed, 8 Sep 2021 12:11:42 +0200 Subject: [PATCH 25/37] fix: correct comment indentation --- deploy/helm/sumologic/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index fb35beade6..11b3cbd17e 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -3040,8 +3040,8 @@ logs: ## You can specify only one of maxUnavailable and minAvailable in a single PodDisruptionBudget podDisruptionBudget: minAvailable: 2 - ## To use maxUnavailable, set minAvailable to null and uncomment the below: - # maxUnavailable: 1 + ## To use maxUnavailable, set minAvailable to null and uncomment the below: + # maxUnavailable: 1 ## Persist data to a persistent volume; When enabled, fluentd uses the file buffer instead of memory buffer. ## ToDo: implement on otelcol side From 279f46adb436ba6c9b601d56d3ccbe4abffb3200 Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Wed, 8 Sep 2021 12:18:38 +0200 Subject: [PATCH 26/37] cleanup: remove incorrect comment --- deploy/helm/sumologic/values.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 11b3cbd17e..f37f4777a2 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -3046,8 +3046,6 @@ logs: ## Persist data to a persistent volume; When enabled, fluentd uses the file buffer instead of memory buffer. ## ToDo: implement on otelcol side persistence: - ## After changing this value please follow steps described in: - ## https://github.com/SumoLogic/sumologic-kubernetes-collection/blob/main/deploy/docs/FluentdPersistence.md enabled: true ## If defined, storageClassName: From 75bd20765ed6b874a74b16998191cb017694dbc2 Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Wed, 8 Sep 2021 12:30:10 +0200 Subject: [PATCH 27/37] fix: Move `otelcol.logs` to `otelcol.metadata.logs` --- .../sumologic/conf/logs/otelcol/config.yaml | 2 +- deploy/helm/sumologic/templates/_helpers.tpl | 2 +- deploy/helm/sumologic/values.yaml | 304 +++++++++--------- 3 files changed, 154 insertions(+), 154 deletions(-) diff --git a/deploy/helm/sumologic/conf/logs/otelcol/config.yaml b/deploy/helm/sumologic/conf/logs/otelcol/config.yaml index b8244d90f6..fd405370b6 100644 --- a/deploy/helm/sumologic/conf/logs/otelcol/config.yaml +++ b/deploy/helm/sumologic/conf/logs/otelcol/config.yaml @@ -1 +1 @@ -{{ tpl (toYaml .Values.otelcol.logs.config | replace ": '{{" ": {{" | replace "}}'" "}}") . | nindent 2 }} +{{ tpl (toYaml .Values.otelcol.metadata.logs.config | replace ": '{{" ": {{" | replace "}}'" "}}") . | nindent 2 }} diff --git a/deploy/helm/sumologic/templates/_helpers.tpl b/deploy/helm/sumologic/templates/_helpers.tpl index 50de7815b4..20be697975 100644 --- a/deploy/helm/sumologic/templates/_helpers.tpl +++ b/deploy/helm/sumologic/templates/_helpers.tpl @@ -1114,7 +1114,7 @@ Example Usage: {{- define "logs.otelcol.enabled" -}} {{- $enabled := false -}} {{- if eq .Values.sumologic.logs.enabled true -}} -{{- if and (eq .Values.sumologic.logs.provider "otelcol") (eq .Values.otelcol.logs.enabled true) -}} +{{- if and (eq .Values.sumologic.logs.provider "otelcol") (eq .Values.otelcol.metadata.logs.enabled true) -}} {{- $enabled = true -}} {{- end -}} {{- end -}} diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index f37f4777a2..322d4dd7dd 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -2619,159 +2619,159 @@ otelcol: exporters: - sumologic - logs: - enabled: true - config: - receivers: - fluentforward: - endpoint: 0.0.0.0:24321 - extensions: - health_check: {} - exporters: - sumologic: - log_format: json - endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE} - source_name: "%{k8s.namespace.name}.%{k8s.pod.name}.%{k8s.container.name}" - ## ToDo: Add `-` replacement configuration, by default with `/` - source_category: "%{k8s.namespace.name}/%{k8s.pod.pod_name}" - source_host: '%{k8s.pod.hostname}' - metadata_attributes: - - k8s.* - - pod_.* - - namespace_.* - - - Deployment - - Namespace - - container - - host - - node - - pod - processors: - attributes: - actions: - - action: extract - key: fluent.tag - pattern: ^containers\.var\.log\.containers\.(?P[^_]+)_(?P[^_]+)_(?P.+)-(?P[a-f0-9]{64})\.log$ - - action: delete - key: fluent.tag - - - action: insert - key: k8s.container.id - from_attribute: container_id - - action: delete - key: container_id - - - action: insert - key: k8s.pod.name - from_attribute: k8s_pod_name - - action: delete - key: k8s_pod_name - - - action: insert - key: k8s.namespace.name - from_attribute: k8s_namespace - - action: delete - key: k8s_namespace - - - action: insert - key: k8s.container.name - from_attribute: k8s_container_name - - action: delete - key: k8s_container_name - groupbyattrs: - keys: - - k8s.container.id - - k8s.container.name - - k8s.instance.id - - k8s.namespace.name - - k8s.pod.name - - k8s.pod.uid - - ## The memory_limiter processor is used to prevent out of memory situations on the collector. - memory_limiter: - ## check_interval is the time between measurements of memory usage for the - ## purposes of avoiding going over the limits. Defaults to zero, so no - ## checks will be performed. Values below 1 second are not recommended since - ## it can result in unnecessary CPU consumption. - check_interval: 5s - - ## Maximum amount of memory, in MiB, targeted to be allocated by the process heap. - ## Note that typically the total memory usage of process will be about 50MiB higher - ## than this value. - limit_mib: 1900 - - ## The batch processor accepts spans and places them into batches grouped by node and resource - batch: - ## Number of spans after which a batch will be sent regardless of time - send_batch_size: 256 - ## Time duration after which a batch will be sent regardless of size - timeout: 5s - k8s_tagger: - ## Has to be false to enrich metadata - passthrough: false - owner_lookup_enabled: true # To enable fetching additional metadata using `owner` relationship - extract: - metadata: - ## extract the following well-known metadata fields - - containerId - - containerName - - clusterName - - daemonSetName - - deploymentName - - hostName - - namespace - - nodeName - - podId - - podName - - replicaSetName - - serviceName - - statefulSetName - annotations: - - tag_name: "pod_annotations_%s" - key: "*" - namespace_labels: - - tag_name: "namespace_labels_%s" - key: "*" - labels: - - tag_name: "pod_labels_%s" - key: "*" - pod_association: - - from: resource_attribute - name: k8s.pod.uid ## Source processor adds Sumo Logic related metadata - source: - collector: '{{ .Values.sumologic.collectorName | default .Values.sumologic.clusterName | quote }}' - source_name: '{{ .Values.fluentd.logs.containers.sourceName | quote }}' - source_category: '{{ .Values.fluentd.logs.containers.sourceCategory | quote }}' - source_category_prefix: '{{ .Values.fluentd.logs.containers.sourceCategoryPrefix | quote }}' - source_category_replace_dash: '{{ .Values.fluentd.logs.containers.sourceCategoryReplaceDash | quote }}' - exclude_namespace_regex: '{{ include "fluentd.excludeNamespaces" . }}' - exclude_pod_regex: '{{ .Values.fluentd.logs.containers.excludePodRegex | quote }}' - exclude_container_regex: '{{ .Values.fluentd.logs.containers.excludeContainerRegex | quote }}' - exclude_host_regex: '{{ .Values.fluentd.logs.containers.excludeHostRegex | quote }}' - annotation_prefix: "k8s.pod.annotation." - pod_template_hash_key: "k8s.pod.label.pod-template-hash" - pod_name_key: "k8s.pod.pod_name" - namespace_key: "k8s.namespace.name" - pod_key: "k8s.pod.name" - container_key: "k8s.container.name" - source_host_key: "k8s.pod.hostname" - service: + logs: + enabled: true + config: + receivers: + fluentforward: + endpoint: 0.0.0.0:24321 extensions: - - health_check - # - sumologic - pipelines: - logs: - receivers: - - fluentforward - processors: - - memory_limiter - - attributes - - groupbyattrs - - k8s_tagger - - source - - batch - exporters: - - sumologic + health_check: {} + exporters: + sumologic: + log_format: json + endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE} + source_name: "%{k8s.namespace.name}.%{k8s.pod.name}.%{k8s.container.name}" + ## ToDo: Add `-` replacement configuration, by default with `/` + source_category: "%{k8s.namespace.name}/%{k8s.pod.pod_name}" + source_host: '%{k8s.pod.hostname}' + metadata_attributes: + - k8s.* + - pod_.* + - namespace_.* + + - Deployment + - Namespace + - container + - host + - node + - pod + processors: + attributes: + actions: + - action: extract + key: fluent.tag + pattern: ^containers\.var\.log\.containers\.(?P[^_]+)_(?P[^_]+)_(?P.+)-(?P[a-f0-9]{64})\.log$ + - action: delete + key: fluent.tag + + - action: insert + key: k8s.container.id + from_attribute: container_id + - action: delete + key: container_id + + - action: insert + key: k8s.pod.name + from_attribute: k8s_pod_name + - action: delete + key: k8s_pod_name + + - action: insert + key: k8s.namespace.name + from_attribute: k8s_namespace + - action: delete + key: k8s_namespace + + - action: insert + key: k8s.container.name + from_attribute: k8s_container_name + - action: delete + key: k8s_container_name + groupbyattrs: + keys: + - k8s.container.id + - k8s.container.name + - k8s.instance.id + - k8s.namespace.name + - k8s.pod.name + - k8s.pod.uid + + ## The memory_limiter processor is used to prevent out of memory situations on the collector. + memory_limiter: + ## check_interval is the time between measurements of memory usage for the + ## purposes of avoiding going over the limits. Defaults to zero, so no + ## checks will be performed. Values below 1 second are not recommended since + ## it can result in unnecessary CPU consumption. + check_interval: 5s + + ## Maximum amount of memory, in MiB, targeted to be allocated by the process heap. + ## Note that typically the total memory usage of process will be about 50MiB higher + ## than this value. + limit_mib: 1900 + + ## The batch processor accepts spans and places them into batches grouped by node and resource + batch: + ## Number of spans after which a batch will be sent regardless of time + send_batch_size: 256 + ## Time duration after which a batch will be sent regardless of size + timeout: 5s + k8s_tagger: + ## Has to be false to enrich metadata + passthrough: false + owner_lookup_enabled: true # To enable fetching additional metadata using `owner` relationship + extract: + metadata: + ## extract the following well-known metadata fields + - containerId + - containerName + - clusterName + - daemonSetName + - deploymentName + - hostName + - namespace + - nodeName + - podId + - podName + - replicaSetName + - serviceName + - statefulSetName + annotations: + - tag_name: "pod_annotations_%s" + key: "*" + namespace_labels: + - tag_name: "namespace_labels_%s" + key: "*" + labels: + - tag_name: "pod_labels_%s" + key: "*" + pod_association: + - from: resource_attribute + name: k8s.pod.uid ## Source processor adds Sumo Logic related metadata + source: + collector: '{{ .Values.sumologic.collectorName | default .Values.sumologic.clusterName | quote }}' + source_name: '{{ .Values.fluentd.logs.containers.sourceName | quote }}' + source_category: '{{ .Values.fluentd.logs.containers.sourceCategory | quote }}' + source_category_prefix: '{{ .Values.fluentd.logs.containers.sourceCategoryPrefix | quote }}' + source_category_replace_dash: '{{ .Values.fluentd.logs.containers.sourceCategoryReplaceDash | quote }}' + exclude_namespace_regex: '{{ include "fluentd.excludeNamespaces" . }}' + exclude_pod_regex: '{{ .Values.fluentd.logs.containers.excludePodRegex | quote }}' + exclude_container_regex: '{{ .Values.fluentd.logs.containers.excludeContainerRegex | quote }}' + exclude_host_regex: '{{ .Values.fluentd.logs.containers.excludeHostRegex | quote }}' + annotation_prefix: "k8s.pod.annotation." + pod_template_hash_key: "k8s.pod.label.pod-template-hash" + pod_name_key: "k8s.pod.pod_name" + namespace_key: "k8s.namespace.name" + pod_key: "k8s.pod.name" + container_key: "k8s.container.name" + source_host_key: "k8s.pod.hostname" + service: + extensions: + - health_check + # - sumologic + pipelines: + logs: + receivers: + - fluentforward + processors: + - memory_limiter + - attributes + - groupbyattrs + - k8s_tagger + - source + - batch + exporters: + - sumologic deployment: nodeSelector: {} tolerations: [] From 60300b5433c54cee24715819a020ee8d98beedd9 Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Wed, 8 Sep 2021 13:19:04 +0200 Subject: [PATCH 28/37] fix: move `sumologic.logs.provider` to `sumologic.logs.metadata.provider` --- deploy/helm/sumologic/templates/_helpers.tpl | 8 ++++---- deploy/helm/sumologic/templates/checks.txt | 4 ++-- .../sumologic/templates/logs/common/service-headless.yaml | 4 ++-- deploy/helm/sumologic/templates/logs/common/service.yaml | 4 ++-- deploy/helm/sumologic/values.yaml | 7 ++++--- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/deploy/helm/sumologic/templates/_helpers.tpl b/deploy/helm/sumologic/templates/_helpers.tpl index 20be697975..54e838099c 100644 --- a/deploy/helm/sumologic/templates/_helpers.tpl +++ b/deploy/helm/sumologic/templates/_helpers.tpl @@ -297,9 +297,9 @@ helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded {{- end -}} {{- define "sumologic.metadata.name.logs" -}} -{{- if eq .Values.sumologic.logs.provider "fluentd" -}} +{{- if eq .Values.sumologic.logs.metadata.provider "fluentd" -}} {{ template "sumologic.metadata.name.fluentd" . }}-logs -{{- else if eq .Values.sumologic.logs.provider "otelcol" -}} +{{- else if eq .Values.sumologic.logs.metadata.provider "otelcol" -}} {{ template "sumologic.metadata.name.otelcol" . }}-logs {{- end -}} {{- end -}} @@ -1114,7 +1114,7 @@ Example Usage: {{- define "logs.otelcol.enabled" -}} {{- $enabled := false -}} {{- if eq .Values.sumologic.logs.enabled true -}} -{{- if and (eq .Values.sumologic.logs.provider "otelcol") (eq .Values.otelcol.metadata.logs.enabled true) -}} +{{- if and (eq .Values.sumologic.logs.metadata.provider "otelcol") (eq .Values.otelcol.metadata.logs.enabled true) -}} {{- $enabled = true -}} {{- end -}} {{- end -}} @@ -1131,7 +1131,7 @@ Example Usage: {{- define "logs.fluentd.enabled" -}} {{- $enabled := false -}} {{- if eq .Values.sumologic.logs.enabled true -}} -{{- if and (eq .Values.sumologic.logs.provider "fluentd") (eq .Values.fluentd.logs.enabled true) -}} +{{- if and (eq .Values.sumologic.logs.metadata.provider "fluentd") (eq .Values.fluentd.logs.enabled true) -}} {{- $enabled = true -}} {{- end -}} {{- end -}} diff --git a/deploy/helm/sumologic/templates/checks.txt b/deploy/helm/sumologic/templates/checks.txt index a97bb710d9..f49dd7363a 100644 --- a/deploy/helm/sumologic/templates/checks.txt +++ b/deploy/helm/sumologic/templates/checks.txt @@ -11,6 +11,6 @@ {{- end -}} {{/* Check if logs provider is set to either fluentd or otelcol */}} -{{- if not (or (eq .Values.sumologic.logs.provider "fluentd") (eq .Values.sumologic.logs.provider "otelcol")) -}} -{{- fail "\nsumologic.logs.provider should be set to either fluentd or otelcol" -}} +{{- if not (or (eq .Values.sumologic.logs.metadata.provider "fluentd") (eq .Values.sumologic.logs.metadata.provider "otelcol")) -}} +{{- fail "\nsumologic.logs.metadata.provider should be set to either fluentd or otelcol" -}} {{- end -}} diff --git a/deploy/helm/sumologic/templates/logs/common/service-headless.yaml b/deploy/helm/sumologic/templates/logs/common/service-headless.yaml index 2c661c28c3..46ab49d428 100644 --- a/deploy/helm/sumologic/templates/logs/common/service-headless.yaml +++ b/deploy/helm/sumologic/templates/logs/common/service-headless.yaml @@ -6,8 +6,8 @@ metadata: labels: app: {{ template "sumologic.labels.app.logs.service-headless" . }} {{- include "sumologic.labels.logs" . | nindent 4 }} -{{- if (include "service.labels" (dict "Provider" .Values.sumologic.logs.provider "Values" .Values )) }} -{{ include "service.labels" (dict "Provider" .Values.sumologic.logs.provider "Values" .Values ) | indent 4 }} +{{- if (include "service.labels" (dict "Provider" .Values.sumologic.logs.metadata.provider "Values" .Values )) }} +{{ include "service.labels" (dict "Provider" .Values.sumologic.logs.metadata.provider "Values" .Values ) | indent 4 }} {{- end }} spec: selector: diff --git a/deploy/helm/sumologic/templates/logs/common/service.yaml b/deploy/helm/sumologic/templates/logs/common/service.yaml index 0b467e56aa..6b365c3535 100644 --- a/deploy/helm/sumologic/templates/logs/common/service.yaml +++ b/deploy/helm/sumologic/templates/logs/common/service.yaml @@ -7,8 +7,8 @@ metadata: app: {{ template "sumologic.labels.app.logs.service" . }} {{- include "sumologic.labels.scrape.logs" . | nindent 4 }} {{- include "sumologic.labels.common" . | nindent 4 }} -{{- if (include "service.labels" (dict "Provider" .Values.sumologic.logs.provider "Values" .Values )) }} -{{ include "service.labels" (dict "Provider" .Values.sumologic.logs.provider "Values" .Values ) | indent 4 }} +{{- if (include "service.labels" (dict "Provider" .Values.sumologic.logs.metadata.provider "Values" .Values )) }} +{{ include "service.labels" (dict "Provider" .Values.sumologic.logs.metadata.provider "Values" .Values ) | indent 4 }} {{- end }} spec: selector: diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 322d4dd7dd..82908202cc 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -183,9 +183,10 @@ sumologic: ## Set the enabled flag to false for disabling logs ingestion altogether. logs: enabled: true - ## Set provider service (either fluentd or otelcol). - ## Otelcol is experimental and may behave in unexpected way. - provider: fluentd + metadata: + ## Set provider service (either fluentd or otelcol). + ## Otelcol is experimental and may behave in unexpected way. + provider: fluentd ## Fields to be created at Sumo Logic to ensure logs are tagged with ## relevant metadata. From fa22248a71e70460737f2c0d1a20c9e82bd6e899 Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Wed, 8 Sep 2021 13:21:42 +0200 Subject: [PATCH 29/37] fix: fix property name in error message --- deploy/helm/sumologic/templates/checks.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/helm/sumologic/templates/checks.txt b/deploy/helm/sumologic/templates/checks.txt index f49dd7363a..c3933564c4 100644 --- a/deploy/helm/sumologic/templates/checks.txt +++ b/deploy/helm/sumologic/templates/checks.txt @@ -7,7 +7,7 @@ {{/* Check if metrics provider is set to either fluentd or otelcol */}} {{- if not (or (eq .Values.sumologic.metrics.metadata.provider "fluentd") (eq .Values.sumologic.metrics.metadata.provider "otelcol")) -}} -{{- fail "\nsumologic.metrics.provider should be set to either fluentd or otelcol" -}} +{{- fail "\nsumologic.metrics.metadata.provider should be set to either fluentd or otelcol" -}} {{- end -}} {{/* Check if logs provider is set to either fluentd or otelcol */}} From fd3c93b79a03e07c72459c8f1c0e9837a1c71fc5 Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Wed, 8 Sep 2021 13:23:50 +0200 Subject: [PATCH 30/37] refactor: simplify service labels logic for OtelCol These templates are currently only used by OtelCol and not by Fluentd, so the additional logic is not needed at the moment. --- .../sumologic/templates/logs/common/service-headless.yaml | 4 ++-- deploy/helm/sumologic/templates/logs/common/service.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/helm/sumologic/templates/logs/common/service-headless.yaml b/deploy/helm/sumologic/templates/logs/common/service-headless.yaml index 46ab49d428..774009a9c0 100644 --- a/deploy/helm/sumologic/templates/logs/common/service-headless.yaml +++ b/deploy/helm/sumologic/templates/logs/common/service-headless.yaml @@ -6,8 +6,8 @@ metadata: labels: app: {{ template "sumologic.labels.app.logs.service-headless" . }} {{- include "sumologic.labels.logs" . | nindent 4 }} -{{- if (include "service.labels" (dict "Provider" .Values.sumologic.logs.metadata.provider "Values" .Values )) }} -{{ include "service.labels" (dict "Provider" .Values.sumologic.logs.metadata.provider "Values" .Values ) | indent 4 }} +{{- if .Values.otelcol.serviceLabels }} +{{ toYaml .Values.otelcol.serviceLabels | indent 4 }} {{- end }} spec: selector: diff --git a/deploy/helm/sumologic/templates/logs/common/service.yaml b/deploy/helm/sumologic/templates/logs/common/service.yaml index 6b365c3535..4a9179105e 100644 --- a/deploy/helm/sumologic/templates/logs/common/service.yaml +++ b/deploy/helm/sumologic/templates/logs/common/service.yaml @@ -7,8 +7,8 @@ metadata: app: {{ template "sumologic.labels.app.logs.service" . }} {{- include "sumologic.labels.scrape.logs" . | nindent 4 }} {{- include "sumologic.labels.common" . | nindent 4 }} -{{- if (include "service.labels" (dict "Provider" .Values.sumologic.logs.metadata.provider "Values" .Values )) }} -{{ include "service.labels" (dict "Provider" .Values.sumologic.logs.metadata.provider "Values" .Values ) | indent 4 }} +{{- if .Values.otelcol.serviceLabels }} +{{ toYaml .Values.otelcol.serviceLabels | indent 4 }} {{- end }} spec: selector: From 353d48079c85287bc2849a611aa17dd932b063a5 Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Wed, 8 Sep 2021 14:05:03 +0200 Subject: [PATCH 31/37] fix: move `logs.*` to `logs.metadata.*` --- .../sumologic/templates/logs/common/hpa.yaml | 12 +- .../sumologic/templates/logs/common/pdb.yaml | 4 +- .../templates/logs/otelcol/statefulset.yaml | 40 ++--- deploy/helm/sumologic/values.yaml | 143 +++++++++--------- 4 files changed, 100 insertions(+), 99 deletions(-) diff --git a/deploy/helm/sumologic/templates/logs/common/hpa.yaml b/deploy/helm/sumologic/templates/logs/common/hpa.yaml index 07d7ca392b..ba458fac66 100644 --- a/deploy/helm/sumologic/templates/logs/common/hpa.yaml +++ b/deploy/helm/sumologic/templates/logs/common/hpa.yaml @@ -1,4 +1,4 @@ -{{- if and (eq (include "logs.otelcol.enabled" .) "true") (.Values.logs.autoscaling.enabled) }} +{{- if and (eq (include "logs.otelcol.enabled" .) "true") (.Values.logs.metadata.autoscaling.enabled) }} apiVersion: autoscaling/v2beta2 kind: HorizontalPodAutoscaler metadata: @@ -11,21 +11,21 @@ spec: apiVersion: apps/v1 kind: StatefulSet name: {{ template "sumologic.metadata.name.logs.statefulset" . }} - minReplicas: {{ .Values.logs.autoscaling.minReplicas }} - maxReplicas: {{ .Values.logs.autoscaling.maxReplicas }} + minReplicas: {{ .Values.logs.metadata.autoscaling.minReplicas }} + maxReplicas: {{ .Values.logs.metadata.autoscaling.maxReplicas }} metrics: - type: Resource resource: name: cpu target: type: Utilization - averageUtilization: {{ .Values.logs.autoscaling.targetCPUUtilizationPercentage }} -{{- if .Values.logs.autoscaling.targetMemoryUtilizationPercentage }} + averageUtilization: {{ .Values.logs.metadata.autoscaling.targetCPUUtilizationPercentage }} +{{- if .Values.logs.metadata.autoscaling.targetMemoryUtilizationPercentage }} - type: Resource resource: name: memory target: type: Utilization - averageUtilization: {{ .Values.logs.autoscaling.targetMemoryUtilizationPercentage }} + averageUtilization: {{ .Values.logs.metadata.autoscaling.targetMemoryUtilizationPercentage }} {{- end -}} {{- end -}} diff --git a/deploy/helm/sumologic/templates/logs/common/pdb.yaml b/deploy/helm/sumologic/templates/logs/common/pdb.yaml index ebd247e334..d4ade30c7c 100644 --- a/deploy/helm/sumologic/templates/logs/common/pdb.yaml +++ b/deploy/helm/sumologic/templates/logs/common/pdb.yaml @@ -1,5 +1,5 @@ {{- if eq (include "logs.otelcol.enabled" .) "true" }} -{{- if .Values.logs.podDisruptionBudget -}} +{{- if .Values.logs.metadata.podDisruptionBudget -}} apiVersion: policy/v1beta1 kind: PodDisruptionBudget metadata: @@ -8,6 +8,6 @@ spec: selector: matchLabels: app: {{ template "sumologic.labels.app.logs.statefulset" . }} -{{ toYaml .Values.logs.podDisruptionBudget | indent 2 }} +{{ toYaml .Values.logs.metadata.podDisruptionBudget | indent 2 }} {{- end -}} {{- end -}} diff --git a/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml b/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml index acd44d7749..bd578871e4 100644 --- a/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml +++ b/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml @@ -12,8 +12,8 @@ spec: app: {{ template "sumologic.labels.app.logs.pod" . }} serviceName: {{ template "sumologic.metadata.name.logs.service-headless" . }} podManagementPolicy: "Parallel" -{{- if not .Values.logs.autoscaling.enabled }} - replicas: {{ .Values.logs.statefulset.replicaCount }} +{{- if not .Values.logs.metadata.autoscaling.enabled }} + replicas: {{ .Values.logs.metadata.statefulset.replicaCount }} {{- end }} template: metadata: @@ -25,8 +25,8 @@ spec: {{- if .Values.otelcol.podAnnotations }} {{ toYaml .Values.otelcol.podAnnotations | indent 8 }} {{- end }} -{{- if .Values.logs.statefulset.podAnnotations }} -{{ toYaml .Values.logs.statefulset.podAnnotations | indent 8 }} +{{- if .Values.logs.metadata.statefulset.podAnnotations }} +{{ toYaml .Values.logs.metadata.statefulset.podAnnotations | indent 8 }} {{- end }} labels: app: {{ template "sumologic.labels.app.logs.pod" . }} @@ -37,25 +37,25 @@ spec: {{- if .Values.otelcol.podLabels }} {{ toYaml .Values.otelcol.podLabels | indent 8 }} {{- end }} -{{- if .Values.logs.statefulset.podLabels }} -{{ toYaml .Values.logs.statefulset.podLabels | indent 8 }} +{{- if .Values.logs.metadata.statefulset.podLabels }} +{{ toYaml .Values.logs.metadata.statefulset.podLabels | indent 8 }} {{- end }} spec: serviceAccountName: {{ template "sumologic.metadata.name.roles.serviceaccount" . }} -{{- if .Values.logs.statefulset.nodeSelector }} +{{- if .Values.logs.metadata.statefulset.nodeSelector }} nodeSelector: -{{ toYaml .Values.logs.statefulset.nodeSelector | indent 8 }} +{{ toYaml .Values.logs.metadata.statefulset.nodeSelector | indent 8 }} {{- end }} -{{- if .Values.logs.statefulset.tolerations }} +{{- if .Values.logs.metadata.statefulset.tolerations }} tolerations: -{{ toYaml .Values.logs.statefulset.tolerations | indent 8 }} +{{ toYaml .Values.logs.metadata.statefulset.tolerations | indent 8 }} {{- end }} -{{- if or .Values.logs.statefulset.affinity .Values.fluentd.logs.statefulset.podAntiAffinity }} +{{- if or .Values.logs.metadata.statefulset.affinity .Values.fluentd.logs.statefulset.podAntiAffinity }} affinity: -{{- if .Values.logs.statefulset.affinity }} -{{ toYaml .Values.logs.statefulset.affinity | indent 8 }} +{{- if .Values.logs.metadata.statefulset.affinity }} +{{ toYaml .Values.logs.metadata.statefulset.affinity | indent 8 }} {{- end }} -{{- if eq .Values.logs.statefulset.podAntiAffinity "hard"}} +{{- if eq .Values.logs.metadata.statefulset.podAntiAffinity "hard"}} podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: @@ -70,7 +70,7 @@ spec: values: - prometheus-operator-prometheus topologyKey: "kubernetes.io/hostname" -{{else if eq .Values.logs.statefulset.podAntiAffinity "soft"}} +{{else if eq .Values.logs.metadata.statefulset.podAntiAffinity "soft"}} podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 @@ -98,18 +98,18 @@ spec: {{- end }} securityContext: {{- toYaml .Values.otelcol.securityContext | nindent 8 }} - {{- if .Values.logs.statefulset.priorityClassName }} - priorityClassName: {{ .Values.logs.statefulset.priorityClassName | quote }} + {{- if .Values.logs.metadata.statefulset.priorityClassName }} + priorityClassName: {{ .Values.logs.metadata.statefulset.priorityClassName | quote }} {{- end }} containers: - name: otelcol image: {{ .Values.otelcol.statefulset.image.repository }}:{{ .Values.otelcol.statefulset.image.tag }} imagePullPolicy: {{ .Values.otelcol.statefulset.image.pullPolicy }} resources: - {{- toYaml .Values.logs.statefulset.resources | nindent 10 }} - {{- if .Values.logs.statefulset.containers.otelcol.securityContext }} + {{- toYaml .Values.logs.metadata.statefulset.resources | nindent 10 }} + {{- if .Values.logs.metadata.statefulset.containers.otelcol.securityContext }} securityContext: - {{- toYaml .Values.logs.statefulset.containers.otelcol.securityContext | nindent 10 }} + {{- toYaml .Values.logs.metadata.statefulset.containers.otelcol.securityContext | nindent 10 }} {{- end }} ports: {{- if .Values.sumologic.traces.enabled }} diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 82908202cc..c9a7d6984d 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -2981,84 +2981,85 @@ otelcol: ## This section affects only otelcol provider. ## In the future it will be used by fluentd as well. logs: - statefulset: - nodeSelector: {} - tolerations: [] - affinity: {} - ## Acceptable values for podAntiAffinity: - ## soft: specifies preferences that the scheduler will try to enforce but will not guarantee (Default) - ## hard: specifies rules that must be met for a pod to be scheduled onto a node - podAntiAffinity: "soft" - replicaCount: 3 - resources: - limits: - memory: 1Gi - cpu: 1000m - requests: - memory: 768Mi - cpu: 500m - ## Option to define priorityClassName to assign a priority class to pods. - priorityClassName: + metadata: + statefulset: + nodeSelector: {} + tolerations: [] + affinity: {} + ## Acceptable values for podAntiAffinity: + ## soft: specifies preferences that the scheduler will try to enforce but will not guarantee (Default) + ## hard: specifies rules that must be met for a pod to be scheduled onto a node + podAntiAffinity: "soft" + replicaCount: 3 + resources: + limits: + memory: 1Gi + cpu: 1000m + requests: + memory: 768Mi + cpu: 500m + ## Option to define priorityClassName to assign a priority class to pods. + priorityClassName: - ## Add custom labels only to logs sts pods - podLabels: {} - ## Add custom annotations only to logs sts pods - podAnnotations: {} + ## Add custom labels only to logs sts pods + podLabels: {} + ## Add custom annotations only to logs sts pods + podAnnotations: {} - ## Set securityContext for containers running in pods in logs statefulset. - containers: - otelcol: - securityContext: {} + ## Set securityContext for containers running in pods in logs statefulset. + containers: + otelcol: + securityContext: {} - ## Extra Environment Values - allows yaml definitions - # extraEnvVars: - # - name: VALUE_FROM_SECRET - # valueFrom: - # secretKeyRef: - # name: secret_name - # key: secret_key + ## Extra Environment Values - allows yaml definitions + # extraEnvVars: + # - name: VALUE_FROM_SECRET + # valueFrom: + # secretKeyRef: + # name: secret_name + # key: secret_key - # extraVolumes: - # - name: es-certs - # secret: - # defaultMode: 420 - # secretName: es-certs - # extraVolumeMounts: - # - name: es-certs - # mountPath: /certs - # readOnly: true + # extraVolumes: + # - name: es-certs + # secret: + # defaultMode: 420 + # secretName: es-certs + # extraVolumeMounts: + # - name: es-certs + # mountPath: /certs + # readOnly: true - ## Option to turn autoscaling on for logs and specify params for HPA. - ## Autoscaling needs metrics-server to access cpu metrics. - autoscaling: - enabled: false - minReplicas: 3 - maxReplicas: 10 - targetCPUUtilizationPercentage: 50 - # targetMemoryUtilizationPercentage: 50 - - ## Option to specify PodDisrutionBudgets - ## You can specify only one of maxUnavailable and minAvailable in a single PodDisruptionBudget - podDisruptionBudget: - minAvailable: 2 - ## To use maxUnavailable, set minAvailable to null and uncomment the below: - # maxUnavailable: 1 + ## Option to turn autoscaling on for logs and specify params for HPA. + ## Autoscaling needs metrics-server to access cpu metrics. + autoscaling: + enabled: false + minReplicas: 3 + maxReplicas: 10 + targetCPUUtilizationPercentage: 50 + # targetMemoryUtilizationPercentage: 50 - ## Persist data to a persistent volume; When enabled, fluentd uses the file buffer instead of memory buffer. - ## ToDo: implement on otelcol side - persistence: - enabled: true + ## Option to specify PodDisrutionBudgets + ## You can specify only one of maxUnavailable and minAvailable in a single PodDisruptionBudget + podDisruptionBudget: + minAvailable: 2 + ## To use maxUnavailable, set minAvailable to null and uncomment the below: + # maxUnavailable: 1 - ## If defined, storageClassName: - ## If set to "-", storageClassName: "", which disables dynamic provisioning - ## If undefined (the default) or set to null, no storageClassName spec is - ## set, choosing the default provisioner. (gp2 on AWS, standard on - ## GKE, Azure & OpenStack) - ## - # storageClass: "-" - # annotations: {} - accessMode: ReadWriteOnce - size: 10Gi + ## Persist data to a persistent volume; When enabled, fluentd uses the file buffer instead of memory buffer. + ## ToDo: implement on otelcol side + persistence: + enabled: true + + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, Azure & OpenStack) + ## + # storageClass: "-" + # annotations: {} + accessMode: ReadWriteOnce + size: 10Gi ## Configure telegraf-operator ## ref: https://github.com/influxdata/helm-charts/blob/master/charts/telegraf-operator/values.yaml From eda1093733d18ddf748b79d01d9a57f7a678bb61 Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Wed, 8 Sep 2021 14:08:23 +0200 Subject: [PATCH 32/37] docs: Add otelcol logs configuration properties to README --- deploy/helm/sumologic/README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/deploy/helm/sumologic/README.md b/deploy/helm/sumologic/README.md index 27897b0d01..548557b787 100644 --- a/deploy/helm/sumologic/README.md +++ b/deploy/helm/sumologic/README.md @@ -20,6 +20,7 @@ The following table lists the configurable parameters of the Sumo Logic chart an | `sumologic.cleanupEnabled` | If enabled, a pre-delete hook will destroy Kubernetes secret and Sumo Logic Collector. | `false` | | `sumologic.logs.enabled` | Set the enabled flag to false for disabling logs ingestion altogether. | `true` | | `sumologic.metrics.enabled` | Set the enabled flag to false for disabling metrics ingestion altogether. | `true` | +| `sumologic.logs.metadata.provider` | Set provider to use for logs forwarding and metadata enrichment. Can be either otelcol or fluentd. | `fluentd` | | `sumologic.metrics.metadata.provider` | Set provider to use for metrics forwarding and metadata enrichment. Can be either otelcol or fluentd. | `fluentd` | | `sumologic.traces.enabled` | Set the enabled flag to true to enable tracing ingestion. _Tracing must be enabled for the account first. Please contact your Sumo representative for activation details_ | `false` | | `sumologic.envFromSecret` | If enabled, accessId and accessKey will be sourced from Secret Name given. Be sure to include at least the following env variables in your secret (1) SUMOLOGIC_ACCESSID, (2) SUMOLOGIC_ACCESSKEY | `sumo-api-secret` | @@ -283,8 +284,33 @@ The following table lists the configurable parameters of the Sumo Logic chart an | `otelcol.statefulset.image.repository` | Image repository for otelcol docker container. | `public.ecr.aws/sumologic/sumologic-otel-collector` | | `otelcol.statefulset.image.tag` | Image tag for otelcol docker container. | `0.0.18` | | `otelcol.statefulset.image.pullPolicy` | Image pullPolicy for otelcol docker container. | `IfNotPresent` | +| `otelcol.metadata.logs.enabled` | Flag to control deploying the otelcol logs statefulsets. | `true` | +| `otelcol.metadata.logs.config` | Configuration for logs otelcol. | [otelcol.metadata.metrics.config in values.yaml](./values.yaml#L2466-L1613) | | `otelcol.metadata.metrics.enabled` | Flag to control deploying the otelcol metrics statefulsets. | `true` | | `otelcol.metadata.metrics.logLevel` | Flag to control logging level for OpenTelemetry Collector for metrics. Can be `debug`, `info`, `warn`, `error`, `dpanic`, `panic`, `fatal`. statefulsets. | `info` | | `otelcol.metadata.metrics.config` | Configuration for metrics otelcol. | [otelcol.metadata.metrics.config in values.yaml](./values.yaml#L2466-L1613) | +| `logs.metadata.statefulset.nodeSelector` | Node selector for logs metadata enrichment (otelcol) statefulset. | `{}` | +| `logs.metadata.statefulset.tolerations` | Tolerations for logs metadata enrichment (otelcol) statefulset. | `[]` | +| `logs.metadata.statefulset.affinity` | Affinity for logs metadata enrichment (otelcol) statefulset. | `{}` | +| `logs.metadata.statefulset.podAntiAffinity` | PodAntiAffinity for logs metadata enrichment (otelcol) statefulset. | `soft` | +| `logs.metadata.statefulset.replicaCount` | Replica count for logs metadata enrichment (otelcol) statefulset. | `3` | +| `logs.metadata.statefulset.resources` | Resources for logs metadata enrichment (otelcol) statefulset. | `{"limits":{"cpu":1,"memory":"1Gi"},"requests":{"cpu":0.5,"memory":"768Mi"}}` | +| `logs.metadata.statefulset.priorityClassName` | Priority class name for logs metadata enrichment (otelcol) pods. | `Nil` | +| `logs.metadata.statefulset.podLabels` | Additional labels for logs metadata enrichment (otelcol) pods. | `{}` | +| `logs.metadata.statefulset.podAnnotations` | Additional annotations for logs metadata enrichment (otelcol) pods. | `{}` | +| `logs.metadata.statefulset.containers.otelcol.securityContext` | The securityContext configuration for otelcol container for logs metadata enrichment statefulset. | `{}` | +| `logs.metadata.statefulset.extraEnvVars` | Additional environment variables for logs metadata enrichment (otelcol) pods. | `Nil` | +| `logs.metadata.statefulset.extraVolumes` | Additional volumes for logs metadata enrichment (otelcol) pods. | `Nil` | +| `logs.metadata.statefulset.extraVolumeMounts` | Additional volume mounts for logs metadata enrichment (otelcol) pods. | `Nil` | +| `logs.metadata.autoscaling.enabled` | Option to turn autoscaling on for logs metadata enrichment (otelcol) and specify params for HPA. Autoscaling needs metrics-server to access cpu metrics. | `false` | +| `logs.metadata.autoscaling.minReplicas` | Default min replicas for autoscaling. | `3` | +| `logs.metadata.autoscaling.maxReplicas` | Default max replicas for autoscaling | `10` | +| `logs.metadata.autoscaling.targetCPUUtilizationPercentage` | The desired target CPU utilization for autoscaling. | `50` | +| `logs.metadata.autoscaling.targetMemoryUtilizationPercentage` | The desired target memory utilization for autoscaling. | `Nil` | +| `logs.metadata.podDisruptionBudget` | Pod Disruption Budget for logs metadata enrichment (otelcol) statefulset. | `{"minAvailable": 2}` | +| `logs.metadata.persistence.enabled` | Persist data to a persistent volume. This is under development. | N/A (is going to be `true`) | +| `logs.metadata.persistence.storageClass` | If defined, storageClassName: . If set to "-", storageClassName: "", which disables dynamic provisioning. If undefined (the default) or set to null, no storageClassName spec is set, choosing the default provisioner. (gp2 on AWS, standard on GKE, Azure & OpenStack) | `Nil` | +| `logs.metadata.persistence.accessMode` | The accessMode for persistence. | `ReadWriteOnce` | +| `logs.metadata.persistence.size` | The size needed for persistence. | `10Gi` | | `tailing-sidecar-operator.enabled` | Flag to control deploying Tailing Sidecar Operator Helm sub-chart. | `false` | | `tailing-sidecar-operator.fullnameOverride` | Used to override the chart's full name. | `Nil` | From cd77253572d7363ac3653c3c7080ae6c7fd3378f Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Wed, 8 Sep 2021 15:00:54 +0200 Subject: [PATCH 33/37] docs: fix markdown formatting --- deploy/helm/sumologic/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/helm/sumologic/README.md b/deploy/helm/sumologic/README.md index 548557b787..39e0d9d19f 100644 --- a/deploy/helm/sumologic/README.md +++ b/deploy/helm/sumologic/README.md @@ -302,7 +302,7 @@ The following table lists the configurable parameters of the Sumo Logic chart an | `logs.metadata.statefulset.extraEnvVars` | Additional environment variables for logs metadata enrichment (otelcol) pods. | `Nil` | | `logs.metadata.statefulset.extraVolumes` | Additional volumes for logs metadata enrichment (otelcol) pods. | `Nil` | | `logs.metadata.statefulset.extraVolumeMounts` | Additional volume mounts for logs metadata enrichment (otelcol) pods. | `Nil` | -| `logs.metadata.autoscaling.enabled` | Option to turn autoscaling on for logs metadata enrichment (otelcol) and specify params for HPA. Autoscaling needs metrics-server to access cpu metrics. | `false` | +| `logs.metadata.autoscaling.enabled` | Option to turn autoscaling on for logs metadata enrichment (otelcol) and specify params for HPA. Autoscaling needs metrics-server to access cpu metrics. | `false` | | `logs.metadata.autoscaling.minReplicas` | Default min replicas for autoscaling. | `3` | | `logs.metadata.autoscaling.maxReplicas` | Default max replicas for autoscaling | `10` | | `logs.metadata.autoscaling.targetCPUUtilizationPercentage` | The desired target CPU utilization for autoscaling. | `50` | From ab0758d37657069e239ce6ffbf16c535e8e00405 Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Wed, 8 Sep 2021 15:56:35 +0200 Subject: [PATCH 34/37] docs: add link to OT distro configuration --- deploy/helm/sumologic/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/helm/sumologic/README.md b/deploy/helm/sumologic/README.md index 39e0d9d19f..a9df226aa5 100644 --- a/deploy/helm/sumologic/README.md +++ b/deploy/helm/sumologic/README.md @@ -285,10 +285,10 @@ The following table lists the configurable parameters of the Sumo Logic chart an | `otelcol.statefulset.image.tag` | Image tag for otelcol docker container. | `0.0.18` | | `otelcol.statefulset.image.pullPolicy` | Image pullPolicy for otelcol docker container. | `IfNotPresent` | | `otelcol.metadata.logs.enabled` | Flag to control deploying the otelcol logs statefulsets. | `true` | -| `otelcol.metadata.logs.config` | Configuration for logs otelcol. | [otelcol.metadata.metrics.config in values.yaml](./values.yaml#L2466-L1613) | +| `otelcol.metadata.logs.config` | Configuration for logs otelcol. See also https://github.com/SumoLogic/sumologic-otel-collector/blob/main/docs/Configuration.md. | [otelcol.metadata.metrics.config in values.yaml](./values.yaml#L2466-L1613) | | `otelcol.metadata.metrics.enabled` | Flag to control deploying the otelcol metrics statefulsets. | `true` | | `otelcol.metadata.metrics.logLevel` | Flag to control logging level for OpenTelemetry Collector for metrics. Can be `debug`, `info`, `warn`, `error`, `dpanic`, `panic`, `fatal`. statefulsets. | `info` | -| `otelcol.metadata.metrics.config` | Configuration for metrics otelcol. | [otelcol.metadata.metrics.config in values.yaml](./values.yaml#L2466-L1613) | +| `otelcol.metadata.metrics.config` | Configuration for metrics otelcol. See also https://github.com/SumoLogic/sumologic-otel-collector/blob/main/docs/Configuration.md. | [otelcol.metadata.metrics.config in values.yaml](./values.yaml#L2466-L1613) | | `logs.metadata.statefulset.nodeSelector` | Node selector for logs metadata enrichment (otelcol) statefulset. | `{}` | | `logs.metadata.statefulset.tolerations` | Tolerations for logs metadata enrichment (otelcol) statefulset. | `[]` | | `logs.metadata.statefulset.affinity` | Affinity for logs metadata enrichment (otelcol) statefulset. | `{}` | From 0daa72f1907a3bcca734c87533929a7ebebd3771 Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Wed, 8 Sep 2021 16:14:38 +0200 Subject: [PATCH 35/37] docs: remove redundant word --- deploy/helm/sumologic/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/helm/sumologic/README.md b/deploy/helm/sumologic/README.md index a9df226aa5..a5b69e2e8c 100644 --- a/deploy/helm/sumologic/README.md +++ b/deploy/helm/sumologic/README.md @@ -287,7 +287,7 @@ The following table lists the configurable parameters of the Sumo Logic chart an | `otelcol.metadata.logs.enabled` | Flag to control deploying the otelcol logs statefulsets. | `true` | | `otelcol.metadata.logs.config` | Configuration for logs otelcol. See also https://github.com/SumoLogic/sumologic-otel-collector/blob/main/docs/Configuration.md. | [otelcol.metadata.metrics.config in values.yaml](./values.yaml#L2466-L1613) | | `otelcol.metadata.metrics.enabled` | Flag to control deploying the otelcol metrics statefulsets. | `true` | -| `otelcol.metadata.metrics.logLevel` | Flag to control logging level for OpenTelemetry Collector for metrics. Can be `debug`, `info`, `warn`, `error`, `dpanic`, `panic`, `fatal`. statefulsets. | `info` | +| `otelcol.metadata.metrics.logLevel` | Flag to control logging level for OpenTelemetry Collector for metrics. Can be `debug`, `info`, `warn`, `error`, `dpanic`, `panic`, `fatal`. | `info` | | `otelcol.metadata.metrics.config` | Configuration for metrics otelcol. See also https://github.com/SumoLogic/sumologic-otel-collector/blob/main/docs/Configuration.md. | [otelcol.metadata.metrics.config in values.yaml](./values.yaml#L2466-L1613) | | `logs.metadata.statefulset.nodeSelector` | Node selector for logs metadata enrichment (otelcol) statefulset. | `{}` | | `logs.metadata.statefulset.tolerations` | Tolerations for logs metadata enrichment (otelcol) statefulset. | `[]` | From e935ec5662daed04cb8ac3dfa6e3710042d18e5c Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Wed, 8 Sep 2021 16:17:40 +0200 Subject: [PATCH 36/37] feat(configuration): add logLevel option for logs otelcol --- deploy/helm/sumologic/README.md | 1 + deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml | 3 +++ deploy/helm/sumologic/values.yaml | 1 + 3 files changed, 5 insertions(+) diff --git a/deploy/helm/sumologic/README.md b/deploy/helm/sumologic/README.md index a5b69e2e8c..d2e2ceaab9 100644 --- a/deploy/helm/sumologic/README.md +++ b/deploy/helm/sumologic/README.md @@ -285,6 +285,7 @@ The following table lists the configurable parameters of the Sumo Logic chart an | `otelcol.statefulset.image.tag` | Image tag for otelcol docker container. | `0.0.18` | | `otelcol.statefulset.image.pullPolicy` | Image pullPolicy for otelcol docker container. | `IfNotPresent` | | `otelcol.metadata.logs.enabled` | Flag to control deploying the otelcol logs statefulsets. | `true` | +| `otelcol.metadata.logs.logLevel` | Flag to control logging level for OpenTelemetry Collector for logs. Can be `debug`, `info`, `warn`, `error`, `dpanic`, `panic`, `fatal`. | `info` | | `otelcol.metadata.logs.config` | Configuration for logs otelcol. See also https://github.com/SumoLogic/sumologic-otel-collector/blob/main/docs/Configuration.md. | [otelcol.metadata.metrics.config in values.yaml](./values.yaml#L2466-L1613) | | `otelcol.metadata.metrics.enabled` | Flag to control deploying the otelcol metrics statefulsets. | `true` | | `otelcol.metadata.metrics.logLevel` | Flag to control logging level for OpenTelemetry Collector for metrics. Can be `debug`, `info`, `warn`, `error`, `dpanic`, `panic`, `fatal`. | `info` | diff --git a/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml b/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml index bd578871e4..e2d1d2b952 100644 --- a/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml +++ b/deploy/helm/sumologic/templates/logs/otelcol/statefulset.yaml @@ -105,6 +105,9 @@ spec: - name: otelcol image: {{ .Values.otelcol.statefulset.image.repository }}:{{ .Values.otelcol.statefulset.image.tag }} imagePullPolicy: {{ .Values.otelcol.statefulset.image.pullPolicy }} + args: + - --log-level={{ .Values.otelcol.metadata.logs.logLevel }} + - --config=/etc/otel/config.yaml resources: {{- toYaml .Values.logs.metadata.statefulset.resources | nindent 10 }} {{- if .Values.logs.metadata.statefulset.containers.otelcol.securityContext }} diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index c9a7d6984d..e692359f66 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -2622,6 +2622,7 @@ otelcol: logs: enabled: true + logLevel: info config: receivers: fluentforward: From 471f869513cbf6ad77dff5903adf7a04a2d36d9a Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Thu, 9 Sep 2021 09:49:14 +0200 Subject: [PATCH 37/37] fix: use pod/namespace name in logs enrichment otelcol --- deploy/helm/sumologic/values.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index e692359f66..da016815c1 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -2738,8 +2738,7 @@ otelcol: - tag_name: "pod_labels_%s" key: "*" pod_association: - - from: resource_attribute - name: k8s.pod.uid ## Source processor adds Sumo Logic related metadata + - from: build_hostname source: collector: '{{ .Values.sumologic.collectorName | default .Values.sumologic.clusterName | quote }}' source_name: '{{ .Values.fluentd.logs.containers.sourceName | quote }}'