feat(otellogs/systemd): add support for systemd logs to otellogs (#2364)

* fix(logs/metadata): fix logs metadata for systemd Signed-off-by: Dominik Rosiek <drosiek@sumologic.com> * feat(otellogs/systemd): add support for systemd logs to otellogs Signed-off-by: Dominik Rosiek <drosiek@sumologic.com> * chore(changelog): update Signed-off-by: Dominik Rosiek <drosiek@sumologic.com> * fix(logs/systemd): adjust otellogs output to fluent output Signed-off-by: Dominik Rosiek <drosiek@sumologic.com> * test(logs/systemd): add tests for otellogs Signed-off-by: Dominik Rosiek <drosiek@sumologic.com> * test: update template tests Signed-off-by: Dominik Rosiek <drosiek@sumologic.com> * test: update template tests Signed-off-by: Dominik Rosiek <drosiek@sumologic.com> * refactor: rename exporter Signed-off-by: Dominik Rosiek <drosiek@sumologic.com> * test: update template tests Signed-off-by: Dominik Rosiek <drosiek@sumologic.com> * test: fix test by mounting journal logs Signed-off-by: Dominik Rosiek <drosiek@sumologic.com> * test: update template tests Signed-off-by: Dominik Rosiek <drosiek@sumologic.com> * Update tests/helm/logs_otc_daemonset/static/complex.output.yaml * feat: use common pipeline for fluent and otelcol systemd Signed-off-by: Dominik Rosiek <drosiek@sumologic.com> * Update deploy/helm/sumologic/values.yaml * test: update template tests Signed-off-by: Dominik Rosiek <drosiek@sumologic.com> * feat: add separate pipelines for otlp systemd logs Signed-off-by: Dominik Rosiek <drosiek@sumologic.com> * Apply suggestions from code review * feat(vagrant): update Signed-off-by: Dominik Rosiek <drosiek@sumologic.com> * fix: dix integration tests Signed-off-by: Dominik Rosiek <drosiek@sumologic.com>
SumoLogic · Jul 12, 2022 · c9f76d3 · c9f76d3
1 parent 6abf6ec
commit c9f76d3
Show file tree

Hide file tree

Showing 11 changed files with 415 additions and 1 deletion.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,13 +10,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - feat(otellogs): add additional volumes and env configs [#2410]
+- feat(otellogs/systemd): add support for systemd logs to otellogs [#2364]
 
 ### Changed
 
 [#2422]: feat(metadata): upgrade otelcol to v0.54.0-sumo-0 [#2422]
 
 [#2410]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2410
 [#2422]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2422
+[#2364]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2364
 [Unreleased]: https://github.com/SumoLogic/sumologic-kubernetes-collection/compare/v2.11.0...main
 
 ## [v2.11.0]

diff --git a/deploy/helm/sumologic/templates/logs/collector/otelcol/daemonset.yaml b/deploy/helm/sumologic/templates/logs/collector/otelcol/daemonset.yaml
@@ -77,6 +77,9 @@ spec:
           readOnly: true
         - mountPath: {{ .Values.otellogs.config.extensions.file_storage.directory }}
           name: file-storage
+        - mountPath: /var/log/journal
+          name: varlogjournal
+          readOnly: true
 {{- if .Values.otellogs.daemonset.extraVolumeMounts }}
 {{ toYaml .Values.otellogs.daemonset.extraVolumeMounts | indent 8 }}
 {{- end }}
@@ -137,6 +140,10 @@ spec:
           path: /var/lib/otc
           type: DirectoryOrCreate
         name: file-storage
+      - hostPath:
+          path: /var/log/journal/
+          type: ""
+        name: varlogjournal
 {{- if .Values.otellogs.daemonset.extraVolumes }}
 {{ toYaml .Values.otellogs.daemonset.extraVolumes | indent 6 }}
 {{- end }}

diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml
@@ -4138,6 +4138,10 @@ metadata:
           json_logs:
             add_timestamp: true
             timestamp_key: timestamp
+            ## use flatten_body, but OTLP won't require any flattening
+            ## fluent based logs will be all send as record attributes
+            ## otellogs based logs will be all send as body attributes
+            flatten_body: true
           endpoint: ${SUMO_ENDPOINT_DEFAULT_LOGS_SOURCE}
           source_name: "%{_sourceName}"
           source_category: "%{_sourceCategory}"
@@ -4385,6 +4389,11 @@ metadata:
           source_category: '{{ .Values.fluentd.logs.systemd.sourceCategory | quote }}'
           source_category_prefix: '{{ .Values.fluentd.logs.systemd.sourceCategoryPrefix | quote }}'
           source_category_replace_dash: '{{ .Values.fluentd.logs.systemd.sourceCategoryReplaceDash | quote }}'
+        ## Remove all attributes, so body won't by nested by SumoLogic receiver in case of using otlp format
+        transform/remove_attributes:
+          logs:
+            queries:
+              - limit(attributes, 0)
 
         ## kubelet related processors
         filter/include_kubelet:
@@ -4488,6 +4497,53 @@ metadata:
               - batch
             exporters:
               - sumologic/systemd
+          ## This is the same pipeline like logs/fluent/systemd, but with the following changes:
+          ## - otlp receiver instead of fluentforward
+          ## - added transform/remove_attributes processor
+          logs/otlp/systemd:
+            receivers:
+              - otlp
+            processors:
+              - memory_limiter
+              - filter/include_fluent_tag_host
+              - filter/include_systemd
+              - filter/exclude_kubelet
+              - filter/exclude_systemd_syslog
+              - filter/exclude_systemd_hostname
+              - filter/exclude_systemd_priority
+              - filter/exclude_systemd_unit
+              - attributes/extract_systemd_source_fields
+              - attributes/remove_fluent_tag
+              - groupbyattrs/systemd
+              - resource/add_cluster
+              - source/systemd
+              - transform/remove_attributes
+              - batch
+            exporters:
+              - sumologic/systemd
+          ## This is the same pipeline like logs/fluent/kubelet, but with the following changes:
+          ## - otlp receiver instead of fluentforward
+          ## - added transform/remove_attributes processor
+          logs/otlp/kubelet:
+            receivers:
+              - otlp
+            processors:
+              - memory_limiter
+              - filter/include_fluent_tag_host
+              - filter/include_kubelet
+              - filter/exclude_kubelet_syslog
+              - filter/exclude_kubelet_hostname
+              - filter/exclude_kubelet_priority
+              - filter/exclude_kubelet_unit
+              - attributes/extract_systemd_source_fields
+              - attributes/remove_fluent_tag
+              - groupbyattrs/systemd
+              - resource/add_cluster
+              - source/kubelet
+              - transform/remove_attributes
+              - batch
+            exporters:
+              - sumologic/systemd
     statefulset:
       nodeSelector: {}
       tolerations: []
@@ -4849,6 +4905,14 @@ otellogs:
             - batch
           exporters:
             - otlphttp
+        logs/systemd:
+          receivers:
+            - journald
+          processors:
+            - logstransform/systemd
+            - batch
+          exporters:
+            - otlphttp
     receivers:
       filelog/containers:
         include:
@@ -5014,6 +5078,59 @@ otellogs:
           - type: move
             from: body.log
             to: body
+      journald:
+        directory: /var/log/journal
+        ## This is not a full equivalent of fluent-bit filtering as fluent-bit filters by `_SYSTEMD_UNIT`
+        ## Here is filtering by `UNIT`
+        units:
+          - addon-config.service
+          - addon-run.service
+          - cfn-etcd-environment.service
+          - cfn-signal.service
+          - clean-ca-certificates.service
+          - containerd.service
+          - coreos-metadata.service
+          - coreos-setup-environment.service
+          - coreos-tmpfiles.service
+          - dbus.service
+          - docker.service
+          - efs.service
+          - etcd-member.service
+          - etcd.service
+          - etcd2.service
+          - etcd3.service
+          - etcdadm-check.service
+          - etcdadm-reconfigure.service
+          - etcdadm-save.service
+          - etcdadm-update-status.service
+          - flanneld.service
+          - format-etcd2-volume.service
+          - kube-node-taint-and-uncordon.service
+          - kubelet.service
+          - ldconfig.service
+          - locksmithd.service
+          - logrotate.service
+          - lvm2-monitor.service
+          - mdmon.service
+          - nfs-idmapd.service
+          - nfs-mountd.service
+          - nfs-server.service
+          - nfs-utils.service
+          - node-problem-detector.service
+          - ntp.service
+          - oem-cloudinit.service
+          - rkt-gc.service
+          - rkt-metadata.service
+          - rpc-idmapd.service
+          - rpc-mountd.service
+          - rpc-statd.service
+          - rpcbind.service
+          - set-aws-environment.service
+          - system-cloudinit.service
+          - systemd-timesyncd.service
+          - update-ca-certificates.service
+          - user-cloudinit.service
+          - var-lib-etcd2.service
     exporters:
       otlphttp:
         endpoint: http://${LOGS_METADATA_SVC}.${NAMESPACE}.svc.cluster.local:4318
@@ -5026,6 +5143,31 @@ otellogs:
         send_batch_size: 10_240
         ## Time duration after which a batch will be sent regardless of size
         timeout: 1s
+      ## copy _SYSTEMD_UNIT, SYSLOG_FACILITY, _HOSTNAME and PRIORITY from body to attributes
+      ## so they can be used by metadata processors same way like for fluentd
+      ## build fluent.tag attribute as `host.{_SYSTEMD_UNIT}`
+      logstransform/systemd:
+        operators:
+          - type: copy
+            from: body._SYSTEMD_UNIT
+            to: attributes._SYSTEMD_UNIT
+          - type: copy
+            from: body.SYSLOG_FACILITY
+            to: attributes.SYSLOG_FACILITY
+          - type: copy
+            from: body._HOSTNAME
+            to: attributes._HOSTNAME
+          - type: copy
+            from: body.PRIORITY
+            to: attributes.PRIORITY
+          - type: add
+            field: attributes["fluent.tag"]
+            value: EXPR("host." + attributes["_SYSTEMD_UNIT"])
+          ## Removes __CURSOR and __MONOTONIC_TIMESTAMP keys from body
+          - type: remove
+            field: body.__CURSOR
+          - type: remove
+            field: body.__MONOTONIC_TIMESTAMP
   daemonset:
     ## Set securityContext for containers running in pods in log collector daemonset
     securityContext:

diff --git a/tests/helm/logs_otc/static/basic.output.yaml b/tests/helm/logs_otc/static/basic.output.yaml
@@ -25,6 +25,27 @@ data:
       batch:
         send_batch_size: 10240
         timeout: 1s
+      logstransform/systemd:
+        operators:
+        - from: body._SYSTEMD_UNIT
+          to: attributes._SYSTEMD_UNIT
+          type: copy
+        - from: body.SYSLOG_FACILITY
+          to: attributes.SYSLOG_FACILITY
+          type: copy
+        - from: body._HOSTNAME
+          to: attributes._HOSTNAME
+          type: copy
+        - from: body.PRIORITY
+          to: attributes.PRIORITY
+          type: copy
+        - field: attributes["fluent.tag"]
+          type: add
+          value: EXPR("host." + attributes["_SYSTEMD_UNIT"])
+        - field: body.__CURSOR
+          type: remove
+        - field: body.__MONOTONIC_TIMESTAMP
+          type: remove
     receivers:
       filelog/containers:
         fingerprint_size: 17408
@@ -121,6 +142,57 @@ data:
           to: body
           type: move
         start_at: beginning
+      journald:
+        directory: /var/log/journal
+        units:
+        - addon-config.service
+        - addon-run.service
+        - cfn-etcd-environment.service
+        - cfn-signal.service
+        - clean-ca-certificates.service
+        - containerd.service
+        - coreos-metadata.service
+        - coreos-setup-environment.service
+        - coreos-tmpfiles.service
+        - dbus.service
+        - docker.service
+        - efs.service
+        - etcd-member.service
+        - etcd.service
+        - etcd2.service
+        - etcd3.service
+        - etcdadm-check.service
+        - etcdadm-reconfigure.service
+        - etcdadm-save.service
+        - etcdadm-update-status.service
+        - flanneld.service
+        - format-etcd2-volume.service
+        - kube-node-taint-and-uncordon.service
+        - kubelet.service
+        - ldconfig.service
+        - locksmithd.service
+        - logrotate.service
+        - lvm2-monitor.service
+        - mdmon.service
+        - nfs-idmapd.service
+        - nfs-mountd.service
+        - nfs-server.service
+        - nfs-utils.service
+        - node-problem-detector.service
+        - ntp.service
+        - oem-cloudinit.service
+        - rkt-gc.service
+        - rkt-metadata.service
+        - rpc-idmapd.service
+        - rpc-mountd.service
+        - rpc-statd.service
+        - rpcbind.service
+        - set-aws-environment.service
+        - system-cloudinit.service
+        - systemd-timesyncd.service
+        - update-ca-certificates.service
+        - user-cloudinit.service
+        - var-lib-etcd2.service
     service:
       extensions:
       - health_check
@@ -134,6 +206,14 @@ data:
           - batch
           receivers:
           - filelog/containers
+        logs/systemd:
+          exporters:
+          - otlphttp
+          processors:
+          - logstransform/systemd
+          - batch
+          receivers:
+          - journald
       telemetry:
         logs:
           level: info
diff --git a/tests/helm/logs_otc_daemonset/static/basic.output.yaml b/tests/helm/logs_otc_daemonset/static/basic.output.yaml
@@ -59,6 +59,9 @@ spec:
           readOnly: true
         - mountPath: /var/lib/storage/otc
           name: file-storage
+        - mountPath: /var/log/journal
+          name: varlogjournal
+          readOnly: true
         env:
         - name: LOGS_METADATA_SVC
           valueFrom:
@@ -111,4 +114,8 @@ spec:
           path: /var/lib/otc
           type: DirectoryOrCreate
         name: file-storage
+      - hostPath:
+          path: /var/log/journal/
+          type: ""
+        name: varlogjournal
       serviceAccountName: RELEASE-NAME-sumologic-otelcol-logs-collector
diff --git a/tests/helm/logs_otc_daemonset/static/complex.output.yaml b/tests/helm/logs_otc_daemonset/static/complex.output.yaml
@@ -69,6 +69,9 @@ spec:
           readOnly: true
         - mountPath: /var/lib/storage/otc
           name: file-storage
+        - mountPath: /var/log/journal
+          name: varlogjournal
+          readOnly: true
         - mountPath: /certs
           name: es-certs
           readOnly: true
@@ -130,6 +133,10 @@ spec:
           path: /var/lib/otc
           type: DirectoryOrCreate
         name: file-storage
+      - hostPath:
+          path: /var/log/journal/
+          type: ""
+        name: varlogjournal
       - name: es-certs
         secret:
           defaultMode: 420