-
Notifications
You must be signed in to change notification settings - Fork 185
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: refactor OT logging configuration
- Loading branch information
1 parent
c9f76d3
commit 4d208fb
Showing
12 changed files
with
358 additions
and
329 deletions.
There are no files selected for viewing
246 changes: 245 additions & 1 deletion
246
deploy/helm/sumologic/conf/logs/collector/otelcol/config.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1 +1,245 @@ | ||
| {{ tpl (toYaml .Values.otellogs.config | replace ": '{{" ": {{" | replace "}}'" "}}") . | nindent 2 }} | ||
| extensions: | ||
| health_check: {} | ||
| {{- if .Values.sumologic.logs.persistence.enabled }} | ||
| file_storage: | ||
| directory: {{ .Values.sumologic.logs.persistence.storageDirectory }} | ||
| timeout: 10s | ||
| {{- end }} | ||
| pprof: {} | ||
| service: | ||
| telemetry: | ||
| logs: | ||
| level: {{ .Values.otellogs.logLevel | quote }} | ||
| extensions: | ||
| - health_check | ||
| {{- if .Values.sumologic.logs.persistence.enabled }} | ||
| - file_storage | ||
| {{- end }} | ||
| - pprof | ||
| pipelines: | ||
| logs/containers: | ||
| receivers: | ||
| - filelog/containers | ||
| processors: | ||
| - batch | ||
| exporters: | ||
| - otlphttp | ||
| {{- if .Values.sumologic.logs.systemd.enabled }} | ||
| logs/systemd: | ||
| receivers: | ||
| - journald | ||
| processors: | ||
| - logstransform/systemd | ||
| - batch | ||
| exporters: | ||
| - otlphttp | ||
| {{- end }} | ||
| receivers: | ||
| filelog/containers: | ||
| include: | ||
| - /var/log/pods/*/*/*.log | ||
| start_at: beginning | ||
| ## sets fingerprint_size to 17kb in order to match the longest possible docker line (which by default is 16kb) | ||
| ## we want to include timestamp, which is at the end of the line | ||
| fingerprint_size: 17408 | ||
| include_file_path: true | ||
| include_file_name: false | ||
| operators: | ||
| ## Detect the container runtime log format | ||
| ## Can be: docker-shim, CRI-O and containerd | ||
| - id: get-format | ||
| type: router | ||
| routes: | ||
| - output: parser-docker | ||
| expr: 'body matches "^\\{"' | ||
| - output: parser-crio | ||
| expr: 'body matches "^[^ Z]+ "' | ||
| - output: parser-containerd | ||
| expr: 'body matches "^[^ Z]+Z"' | ||
| ## Parse CRI-O format | ||
| - id: parser-crio | ||
| type: regex_parser | ||
| regex: '^(?P<time>[^ Z]+) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*)( |)(?P<log>.*)$' | ||
| output: merge-cri-lines | ||
| parse_to: body | ||
| timestamp: | ||
| parse_from: body.time | ||
| layout_type: gotime | ||
| layout: '2006-01-02T15:04:05.000000000-07:00' | ||
| ## Parse CRI-Containerd format | ||
| - id: parser-containerd | ||
| type: regex_parser | ||
| regex: '^(?P<time>[^ ^Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*)( |)(?P<log>.*)$' | ||
| output: merge-cri-lines | ||
| parse_to: body | ||
| timestamp: | ||
| parse_from: body.time | ||
| layout: '%Y-%m-%dT%H:%M:%S.%LZ' | ||
| ## Parse docker-shim format | ||
| ## parser-docker interprets the input string as JSON and moves the `time` field from the JSON to Timestamp field in the OTLP log | ||
| ## record. | ||
| ## Input Body (string): '{"log":"2001-02-03 04:05:06 first line\n","stream":"stdout","time":"2021-11-25T09:59:13.23887954Z"}' | ||
| ## Output Body (JSON): { "log": "2001-02-03 04:05:06 first line\n", "stream": "stdout" } | ||
| ## Input Timestamp: _empty_ | ||
| ## Output Timestamp: 2021-11-25 09:59:13.23887954 +0000 UTC | ||
| - id: parser-docker | ||
| type: json_parser | ||
| parse_to: body | ||
| output: merge-docker-lines | ||
| timestamp: | ||
| parse_from: body.time | ||
| layout: '%Y-%m-%dT%H:%M:%S.%LZ' | ||
|
|
||
| ## merge-docker-lines stitches back together log lines split by Docker logging driver. | ||
| ## Input Body (JSON): { "log": "2001-02-03 04:05:06 very long li", "stream": "stdout" } | ||
| ## Input Body (JSON): { "log": "ne that was split by the logging driver\n", "stream": "stdout" } | ||
| ## Output Body (JSON): { "log": "2001-02-03 04:05:06 very long line that was split by the logging driver\n","stream":"stdout"} | ||
| - id: merge-docker-lines | ||
| type: recombine | ||
| source_identifier: attributes["log.file.path"] | ||
| output: {{ .Values.sumologic.logs.multiline.enabled | ternary "merge-multiline-logs" "extract-metadata-from-filepath" }} | ||
| combine_field: body.log | ||
| combine_with: "" | ||
| is_last_entry: body.log matches "\n$" | ||
|
|
||
| ## merge-cri-lines stitches back together log lines split by CRI logging drivers. | ||
| ## Input Body (JSON): { "log": "2001-02-03 04:05:06 very long li", "logtag": "P" } | ||
| ## Input Body (JSON): { "log": "ne that was split by the logging driver", "logtag": "F" } | ||
| ## Output Body (JSON): { "log": "2001-02-03 04:05:06 very long line that was split by the logging driver\n", "stream": "stdout" } | ||
| - id: merge-cri-lines | ||
| type: recombine | ||
| source_identifier: attributes["log.file.path"] | ||
| output: {{ .Values.sumologic.logs.multiline.enabled | ternary "merge-multiline-logs" "extract-metadata-from-filepath" }} | ||
| combine_field: body.log | ||
| combine_with: "" | ||
| is_last_entry: body.logtag == "F" | ||
| overwrite_with: newest | ||
|
|
||
| ## merge-multiline-logs merges incoming log records into multiline logs. | ||
| ## Input Body (JSON): { "log": "2001-02-03 04:05:06 first line\n", "stream": "stdout" } | ||
| ## Input Body (JSON): { "log": " second line\n", "stream": "stdout" } | ||
| ## Input Body (JSON): { "log": " third line\n", "stream": "stdout" } | ||
| ## Output Body (JSON): { "log": "2001-02-03 04:05:06 first line\n second line\n third line\n", "stream": "stdout" } | ||
| {{- if .Values.sumologic.logs.multiline.enabled }} | ||
| - id: merge-multiline-logs | ||
| type: recombine | ||
| output: extract-metadata-from-filepath | ||
| source_identifier: attributes["log.file.path"] | ||
| combine_field: body.log | ||
| combine_with: "" | ||
| is_first_entry: body.log matches {{ .Values.sumologic.logs.multiline.start_regex | quote }} | ||
| {{- end }} | ||
|
|
||
| ## extract-metadata-from-filepath extracts data from the `log.file.path` Attribute into the Attributes | ||
| ## Input Attributes: | ||
| ## - log.file.path: '/var/log/pods/default_logger-multiline-4nvg4_aed49747-b541-4a07-8663-f7e1febc47d5/loggercontainer/0.log' | ||
| ## Output Attributes: | ||
| ## - log.file.path: '/var/log/pods/default_logger-multiline-4nvg4_aed49747-b541-4a07-8663-f7e1febc47d5/loggercontainer/0.log' | ||
| ## - container_name: "loggercontainer", | ||
| ## - namespace: "default", | ||
| ## - pod_name: "logger-multiline-4nvg4", | ||
| ## - run_id: "0", | ||
| ## - uid: "aed49747-b541-4a07-8663-f7e1febc47d5" | ||
| ## } | ||
| - id: extract-metadata-from-filepath | ||
| type: regex_parser | ||
| regex: '^.*\/(?P<namespace>[^_]+)_(?P<pod_name>[^_]+)_(?P<uid>[a-f0-9\-]+)\/(?P<container_name>[^\._]+)\/(?P<run_id>\d+)\.log$' | ||
| parse_from: attributes["log.file.path"] | ||
|
|
||
| ## The following actions are being performed: | ||
| ## - renaming attributes | ||
| ## - moving stream from body to attribtues | ||
| ## - using body.log as body | ||
| ## - create fluent.tag attribute in order to route in metadata pods | ||
| ## Input Body (JSON): { | ||
| ## "log": "2001-02-03 04:05:06 loggerlog 1 first line\n", | ||
| ## "stream": "stdout", | ||
| ## } | ||
| ## Output Body (String): "2001-02-03 04:05:06 loggerlog 1 first line\n" | ||
| ## Input Attributes: | ||
| ## - log.file.path: '/var/log/pods/default_logger-multiline-4nvg4_aed49747-b541-4a07-8663-f7e1febc47d5/loggercontainer/0.log' | ||
| ## - container_name: "loggercontainer", | ||
| ## - namespace: "default", | ||
| ## - pod_name: "logger-multiline-4nvg4", | ||
| ## - run_id: "0", | ||
| ## - uid: "aed49747-b541-4a07-8663-f7e1febc47d5" | ||
| ## Output Attributes: | ||
| ## - k8s.container.name: "loggercontainer" | ||
| ## - k8s.namespace.name: "default" | ||
| ## - k8s.pod.name: "logger-multiline-4nvg4" | ||
| ## - k8s.pod.uid: "aed49747-b541-4a07-8663-f7e1febc47d5" | ||
| ## - run_id: "0" | ||
| ## - stream: "stdout" | ||
| ## - fluent.tag: "containers.loggercontainer" | ||
| - id: move-attributes | ||
| type: move | ||
| from: body.stream | ||
| to: attributes["stream"] | ||
| - type: move | ||
| from: attributes.container_name | ||
| to: attributes["k8s.container.name"] | ||
| - type: move | ||
| from: attributes.namespace | ||
| to: attributes["k8s.namespace.name"] | ||
| - type: move | ||
| from: attributes.pod_name | ||
| to: attributes["k8s.pod.name"] | ||
| - type: move | ||
| from: attributes.run_id | ||
| to: attributes["run_id"] | ||
| - type: move | ||
| from: attributes.uid | ||
| to: attributes["k8s.pod.uid"] | ||
| - type: add | ||
| field: attributes["fluent.tag"] | ||
| value: EXPR("containers." + attributes["k8s.container.name"]) | ||
| ## Use remove operator when available in opentelemetry collector: | ||
| ## https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/9524 | ||
| - type: move | ||
| from: attributes["log.file.path"] | ||
| to: body["log.file.path"] | ||
| - type: move | ||
| from: body.log | ||
| to: body | ||
| {{- if .Values.sumologic.logs.systemd.enabled }} | ||
| journald: | ||
| directory: /var/log/journal | ||
| ## This is not a full equivalent of fluent-bit filtering as fluent-bit filters by `_SYSTEMD_UNIT` | ||
| ## Here is filtering by `UNIT` | ||
| units: | ||
| {{ toYaml .Values.sumologic.logs.systemd.units | nindent 6 }} | ||
| {{- end }} | ||
| exporters: | ||
| otlphttp: | ||
| endpoint: http://${LOGS_METADATA_SVC}.${NAMESPACE}.svc.cluster.local:4318 | ||
| processors: | ||
| batch: | ||
| send_batch_size: 10_240 | ||
| timeout: 1s | ||
| ## copy _SYSTEMD_UNIT, SYSLOG_FACILITY, _HOSTNAME and PRIORITY from body to attributes | ||
| ## so they can be used by metadata processors same way like for fluentd | ||
| ## build fluent.tag attribute as `host.{_SYSTEMD_UNIT}` | ||
| {{- if .Values.sumologic.logs.systemd.enabled }} | ||
| logstransform/systemd: | ||
| operators: | ||
| - type: copy | ||
| from: body._SYSTEMD_UNIT | ||
| to: attributes._SYSTEMD_UNIT | ||
| - type: copy | ||
| from: body.SYSLOG_FACILITY | ||
| to: attributes.SYSLOG_FACILITY | ||
| - type: copy | ||
| from: body._HOSTNAME | ||
| to: attributes._HOSTNAME | ||
| - type: copy | ||
| from: body.PRIORITY | ||
| to: attributes.PRIORITY | ||
| - type: add | ||
| field: attributes["fluent.tag"] | ||
| value: EXPR("host." + attributes["_SYSTEMD_UNIT"]) | ||
| ## Removes __CURSOR and __MONOTONIC_TIMESTAMP keys from body | ||
| - type: remove | ||
| field: body.__CURSOR | ||
| - type: remove | ||
| field: body.__MONOTONIC_TIMESTAMP | ||
| {{- end }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
2 changes: 1 addition & 1 deletion
2
deploy/helm/sumologic/templates/logs/collector/otelcol/service.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
2 changes: 1 addition & 1 deletion
2
deploy/helm/sumologic/templates/logs/collector/otelcol/serviceaccount.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.