diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 49c2992c7f..4691c1e6a1 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -2592,7 +2592,7 @@ otelcol: statefulset: image: repository: public.ecr.aws/sumologic/sumologic-otel-collector - tag: 0.0.25-beta.0 + tag: 0.0.27-beta.0 pullPolicy: IfNotPresent metadata: metrics: @@ -2769,7 +2769,7 @@ otelcol: extensions: health_check: {} exporters: - sumologic: + sumologic/containers: log_format: json endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE} ## ToDo: Move sources to sourceprocessor @@ -2789,64 +2789,85 @@ otelcol: - host - node - pod + sumologic/systemd: + log_format: json + endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE} + ## ToDo: Move sources to sourceprocessor + source_name: "%{_sourceName}" + source_category: "%{_sourceCategory}" + source_host: "%{_sourceHost}" + sending_queue: + enabled: true + metadata_attributes: + - _collector + - _sourceCategory + - _sourceHost + - _sourceName + processors: - attributes: + ## Common processors + attributes/remove_fluent_tag: actions: - - action: extract - key: fluent.tag - pattern: ^containers\.var\.log\.containers\.(?P[^_]+)_(?P[^_]+)_(?P.+)-(?P[a-f0-9]{64})\.log$ - action: delete key: fluent.tag + ## The memory_limiter processor is used to prevent out of memory situations on the collector. + memory_limiter: + ## check_interval is the time between measurements of memory usage for the + ## purposes of avoiding going over the limits. Defaults to zero, so no + ## checks will be performed. Values below 1 second are not recommended since + ## it can result in unnecessary CPU consumption. + check_interval: 5s + ## Maximum amount of memory, in MiB, targeted to be allocated by the process heap. + ## Note that typically the total memory usage of process will be about 50MiB higher + ## than this value. + limit_mib: 1900 + ## The batch processor accepts spans and places them into batches grouped by node and resource + batch: + ## Number of spans after which a batch will be sent regardless of time + send_batch_size: 256 + ## Time duration after which a batch will be sent regardless of size + timeout: 5s + + ## Containers related processors + filter/include_fluent_tag_containers: + logs: + include: + match_type: regexp + record_attributes: + - key: fluent.tag + value: containers\..+ + attributes/containers: + actions: + - action: extract + key: fluent.tag + pattern: ^containers\.var\.log\.containers\.(?P[^_]+)_(?P[^_]+)_(?P.+)-(?P[a-f0-9]{64})\.log$ - action: insert key: k8s.container.id from_attribute: container_id - action: delete key: container_id - - action: insert key: k8s.pod.name from_attribute: k8s_pod_name - action: delete key: k8s_pod_name - - action: insert key: k8s.namespace.name from_attribute: k8s_namespace - action: delete key: k8s_namespace - - action: insert key: k8s.container.name from_attribute: k8s_container_name - action: delete key: k8s_container_name - groupbyattrs: + groupbyattrs/containers: keys: - k8s.container.id - k8s.container.name - k8s.namespace.name - k8s.pod.name - - ## The memory_limiter processor is used to prevent out of memory situations on the collector. - memory_limiter: - ## check_interval is the time between measurements of memory usage for the - ## purposes of avoiding going over the limits. Defaults to zero, so no - ## checks will be performed. Values below 1 second are not recommended since - ## it can result in unnecessary CPU consumption. - check_interval: 5s - - ## Maximum amount of memory, in MiB, targeted to be allocated by the process heap. - ## Note that typically the total memory usage of process will be about 50MiB higher - ## than this value. - limit_mib: 1900 - - ## The batch processor accepts spans and places them into batches grouped by node and resource - batch: - ## Number of spans after which a batch will be sent regardless of time - send_batch_size: 256 - ## Time duration after which a batch will be sent regardless of size - timeout: 5s k8s_tagger: ## Has to be false to enrich metadata passthrough: false @@ -2879,7 +2900,7 @@ otelcol: delimiter: "_" pod_association: - from: build_hostname - source: + source/containers: collector: '{{ .Values.sumologic.collectorName | default .Values.sumologic.clusterName | quote }}' source_name: '{{ .Values.fluentd.logs.containers.sourceName | quote }}' source_category: '{{ .Values.fluentd.logs.containers.sourceCategory | quote }}' @@ -2897,6 +2918,41 @@ otelcol: pod_key: "k8s.pod.name" container_key: "k8s.container.name" source_host_key: "k8s.pod.hostname" + + ## Systemd related processors + filter/include_fluent_tag_host: + logs: + include: + match_type: regexp + record_attributes: + - key: fluent.tag + value: host\..+ + filter/include_systemd: + logs: + include: + match_type: regexp + record_attributes: + - key: _SYSTEMD_UNIT + value: .+ + filter/exclude_kubelet: + logs: + exclude: + match_type: service + record_attributes: + - key: _SYSTEMD_UNIT + value: kubelet.service + groupbyattrs/systemd: + keys: + - _SYSTEMD_UNIT + - _HOSTNAME + source/systemd: + collector: '{{ .Values.sumologic.collectorName | default .Values.sumologic.clusterName | quote }}' + source_name: '{{ .Values.fluentd.logs.systemd.sourceName | quote }}' + source_category: '{{ .Values.fluentd.logs.systemd.sourceCategory | quote }}' + source_category_prefix: '{{ .Values.fluentd.logs.systemd.sourceCategoryPrefix | quote }}' + source_category_replace_dash: '{{ .Values.fluentd.logs.systemd.sourceCategoryReplaceDash | quote }}' + source_host_key: "_HOSTNAME" + service: extensions: - health_check @@ -2907,13 +2963,30 @@ otelcol: - fluentforward processors: - memory_limiter - - attributes - - groupbyattrs + - filter/include_fluent_tag_containers + - attributes/containers + - attributes/remove_fluent_tag + - groupbyattrs/containers - k8s_tagger - - source + - source/containers - batch exporters: - - sumologic + - sumologic/containers + logs/systemd: + receivers: + - fluentforward + processors: + - memory_limiter + - filter/include_fluent_tag_host + - attributes/remove_fluent_tag + - filter/include_systemd + - filter/exclude_kubelet + - groupbyattrs/systemd + - source/systemd + - batch + exporters: + - sumologic/systemd + deployment: nodeSelector: {} tolerations: [] diff --git a/vagrant/values.yaml b/vagrant/values.yaml index cb94595c6d..79c87297a1 100644 --- a/vagrant/values.yaml +++ b/vagrant/values.yaml @@ -7,14 +7,14 @@ kube-prometheus-stack: dashboardproviders.yaml: apiVersion: 1 providers: - - name: 'default' - orgId: 1 - folder: '' - type: file - disableDeletion: false - editable: true - options: - path: /var/lib/grafana/dashboards/default + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/default sidecar: image: sha: '' @@ -111,6 +111,7 @@ fluent-bit: Systemd_Filter _SYSTEMD_UNIT=format-etcd2-volume.service Systemd_Filter _SYSTEMD_UNIT=kube-node-taint-and-uncordon.service Systemd_Filter _SYSTEMD_UNIT=kubelet.service + Systemd_Filter _SYSTEMD_UNIT=snap.microk8s.daemon-kubelite.service Systemd_Filter _SYSTEMD_UNIT=ldconfig.service Systemd_Filter _SYSTEMD_UNIT=locksmithd.service Systemd_Filter _SYSTEMD_UNIT=logrotate.service @@ -148,3 +149,65 @@ fluentd: multiline: enabled: false + +otelcol: + metadata: + logs: + config: + processors: + # Filter out receiver-mock logs to prevent snowball effect + filter/exclude_fluent_tag_receiver_mock_container: + logs: + exclude: + match_type: regexp + record_attributes: + - key: fluent.tag + value: containers\.var\.log\.containers\.receiver-mock.* + filter/exclude_systemd_snap_kubelite: + logs: + exclude: + match_type: strict + record_attributes: + - key: _SYSTEMD_UNIT + value: snap.microk8s.daemon-kubelite.service + service: + extensions: + - health_check + # - sumologic + pipelines: + logs: + receivers: + - fluentforward + processors: + - memory_limiter + - filter/include_fluent_tag_containers + + # Vagrant specific + - filter/exclude_fluent_tag_receiver_mock_container + + - attributes/containers + - attributes/remove_fluent_tag + - groupbyattrs/containers + - k8s_tagger + - source/containers + - batch + exporters: + - sumologic/containers + logs/systemd: + receivers: + - fluentforward + processors: + - memory_limiter + - filter/include_fluent_tag_host + + # Vagrant specific + - filter/exclude_fluent_tag_receiver_mock_container + + - attributes/remove_fluent_tag + - filter/include_systemd + - filter/exclude_systemd_snap_kubelite + - groupbyattrs/systemd + - source/systemd + - batch + exporters: + - sumologic/systemd