feat(logs): parse JSON logs

SumoLogic · Jan 13, 2023 · ebd10f4 · ebd10f4
1 parent 4887f32
commit ebd10f4
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 0 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - feat: collect metrics from otelcol event collector [#2754]
 - feat: add option to specify additionalEndpoints for metrics [#2788]
 - chore: upgrade kubernetes-setup to v3.5.0 [#2785]
+- feat(logs): parse JSON logs [#2773]
 
 ### Fixed
 
@@ -43,6 +44,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 [#2788]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2788
 [#2785]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2785
 [#2791]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2791
+[#2773]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2773
 [v1.15.3-sumo-0]: https://github.com/SumoLogic/sumologic-kubernetes-fluentd/releases/tag/v1.15.3-sumo-0
 [Unreleased]: https://github.com/SumoLogic/sumologic-kubernetes-collection/compare/v3.0.0-beta.0...main
 

diff --git a/deploy/helm/sumologic/conf/logs/otelcol/config.yaml b/deploy/helm/sumologic/conf/logs/otelcol/config.yaml
@@ -205,6 +205,14 @@ processors:
     container_annotations:
       enabled: {{ .Values.sumologic.logs.container.perContainerAnnotationsEnabled }}
       prefixes: {{ toJson .Values.sumologic.logs.container.perContainerAnnotationPrefixes }}
+  ## logstransformprocessor is experimental, but the same functionality in transformprocessor doesn't offer robust enough error handling
+  ## TODO: use transformprocessor here after https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16519 is closed
+  logstransform/containers_parse_json:
+    operators:
+      - type: json_parser
+        parse_from: body
+        parse_to: body
+        if: body matches "^{.*"
 {{ end }}
 {{ if .Values.sumologic.logs.systemd.enabled }}
   ## Systemd related processors
@@ -421,6 +429,7 @@ service:
         - k8s_tagger
         - resource/add_cluster
         - source/containers
+        - logstransform/containers_parse_json
         - resource/remove_k8s_pod_pod_name
         - resource/drop_annotations
         - resource/containers_copy_node_to_host

diff --git a/tests/helm/metadata_logs_otc/static/otel.output.yaml b/tests/helm/metadata_logs_otc/static/otel.output.yaml
@@ -222,6 +222,12 @@ data:
         passthrough: false
         pod_association:
         - from: build_hostname
+      logstransform/containers_parse_json:
+        operators:
+        - if: body matches "^{.*"
+          parse_from: body
+          parse_to: body
+          type: json_parser
       memory_limiter:
         check_interval: 5s
         limit_percentage: 75
@@ -305,6 +311,7 @@ data:
           - k8s_tagger
           - resource/add_cluster
           - source/containers
+          - logstransform/containers_parse_json
           - resource/remove_k8s_pod_pod_name
           - resource/drop_annotations
           - resource/containers_copy_node_to_host

diff --git a/tests/helm/metadata_logs_otc/static/templates.output.yaml b/tests/helm/metadata_logs_otc/static/templates.output.yaml
@@ -222,6 +222,12 @@ data:
         passthrough: false
         pod_association:
         - from: build_hostname
+      logstransform/containers_parse_json:
+        operators:
+        - if: body matches "^{.*"
+          parse_from: body
+          parse_to: body
+          type: json_parser
       memory_limiter:
         check_interval: 5s
         limit_percentage: 75
@@ -305,6 +311,7 @@ data:
           - k8s_tagger
           - resource/add_cluster
           - source/containers
+          - logstransform/containers_parse_json
           - resource/remove_k8s_pod_pod_name
           - resource/drop_annotations
           - resource/containers_copy_node_to_host