diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c38010d0e..f7c751b1dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - chore: change Fluent Bit image to `public.ecr.aws/sumologic/fluent-bit:1.6.10-sumo-2`, it is Fluent Bit 1.6.10 with updated dependencies, image repository: https://github.com/SumoLogic/fluent-bit-docker-image [#2254][#2254] +- feat(metadata/otc): upgrade to v0.50.0-sumo-0 [#2251][#2251] ### Fixed @@ -36,6 +37,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 [#2246]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2246 [#2254]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2254 [#2255]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2255 +[#2251]: https://github.com/SumoLogic/sumologic-kubernetes-collection/pull/2251 [Unreleased]: https://github.com/SumoLogic/sumologic-kubernetes-collection/compare/v2.6.0...main ## [v2.7.0][v2.7.0] diff --git a/deploy/docs/Best_Practices.md b/deploy/docs/Best_Practices.md index 8b97732ef0..7a778621c6 100644 --- a/deploy/docs/Best_Practices.md +++ b/deploy/docs/Best_Practices.md @@ -214,9 +214,6 @@ metadata: - action: extract key: fluent.tag pattern: ^containers\.var\.log\.pods\.(?P[^_]+)_(?P[^_]+)_(?P[a-f0-9\-]{36})\.(?P[^\._]+)\.(?P\d+)\.log$ - - action: insert - key: k8s.pod.uid - from_attribute: k8s_uid - action: delete key: k8s_uid - action: delete @@ -1058,7 +1055,7 @@ For [sumologic exporter][sumologic_exporter]: **because once filled in PVC never reduces its fill.** [batch_processor]: https://github.com/open-telemetry/opentelemetry-collector/tree/v0.47.0/processor/batchprocessor#batch-processor -[sumologic_exporter]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/exporter/sumologicexporter#sumo-logic-exporter +[sumologic_exporter]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/exporter/sumologicexporter#sumo-logic-exporter [filling_up_alert]: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup/ ### Compaction diff --git a/deploy/docs/fluentd_otc_comparison.md b/deploy/docs/fluentd_otc_comparison.md index 8740c2029d..3891390d1f 100644 --- a/deploy/docs/fluentd_otc_comparison.md +++ b/deploy/docs/fluentd_otc_comparison.md @@ -40,8 +40,8 @@ | `disable_cookies` | Cookies are not used in Opentelemetry Collector | | `compress` | [exporters.sumologic.compress_encoding][otelcol_sumologic_config] set to `""` | | `compress_encoding` | [exporters.sumologic.compress_encoding][otelcol_sumologic_config] | -| `custom_fields` | [Resource processor][resource_processor] combined with [exporters.sumologic.metadata_attributes][otelcol_sumologic_config] | -| `custom_dimensions` | [Resource processor][resource_processor] combined with [exporters.sumologic.metadata_attributes][otelcol_sumologic_config] | +| `custom_fields` | [Resource processor][resource_processor] | +| `custom_dimensions` | [Resource processor][resource_processor] | Additional behavior: @@ -50,14 +50,14 @@ Additional behavior: | [record[_sumo_metadata][source_name]][source_name_precedence] taking precedence over `source_name` | Can be achieved by separate pipelines | | [record[_sumo_metadata][source_host]][source_host_precedence] taking precedence over `source_host` | Can be achieved by separate pipelines | | [record[_sumo_metadata][source_category]][source_category_precedence] taking precedence over `source_category` | Can be achieved by separate pipelines | -| [record[_sumo_metadata][fields]][fields_base] being base for fields | Can be achieved using [resource processor][resource_processor], separate pipelines and [exporter.sumologic.metadata_attributes][otelcol_sumologic_config] | +| [record[_sumo_metadata][fields]][fields_base] being base for fields | Can be achieved using [resource processor][resource_processor] and separate pipelines | [fields_base]: https://github.com/SumoLogic/fluentd-output-sumologic/blob/1.7.2/lib/fluent/plugin/out_sumologic.rb#L284-L285 [fluentd_output_plugin]: https://github.com/sumologic/fluentd-output-sumologic/tree/1.7.2#configuration -[otelcol_basic_confg]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.47.0-sumo-0/docs/Configuration.md#basic-configuration -[otelcol_proxy]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.47.0-sumo-0/docs/Configuration.md#proxy-support -[otelcol_source_config]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/processor/sourceprocessor#config -[otelcol_sumologic_config]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.47.0-sumo-0/pkg/exporter/sumologicexporter/README.md#sumo-logic-exporter +[otelcol_basic_confg]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.50.0-sumo-0/docs/Configuration.md#basic-configuration +[otelcol_proxy]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.50.0-sumo-0/docs/Configuration.md#proxy-support +[otelcol_source_config]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/processor/sourceprocessor#config +[otelcol_sumologic_config]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.50.0-sumo-0/pkg/exporter/sumologicexporter/README.md#sumo-logic-exporter [otelocl_tls_config]: https://github.com/open-telemetry/opentelemetry-collector/blob/v0.47.0/config/configtls/README.md#tls--mtls-configuration [resource_processor]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/v0.47.0/processor/resourceprocessor#resource-processor [source_category_precedence]: https://github.com/SumoLogic/fluentd-output-sumologic/blob/1.7.2/lib/fluent/plugin/out_sumologic.rb#L278-L279 @@ -70,7 +70,7 @@ In order to receive prometheus data and for their initial processing [telegrafre It should cover [fluent-plugin-datapoint][fluent_plugin_datapoint] functionality and more. [fluent_plugin_datapoint]: https://github.com/SumoLogic/sumologic-kubernetes-fluentd/tree/v1.12.2-sumo-4/fluent-plugin-datapoint -[telegrafreceiver]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/receiver/telegrafreceiver +[telegrafreceiver]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/receiver/telegrafreceiver ### fluent-plugin-protobuf @@ -78,7 +78,7 @@ In order to receive prometheus data and for their initial processing [telegrafre It should cover [fluent_plugin_protobuf][fluent_plugin_protobuf] functionality and more. [fluent_plugin_protobuf]: https://github.com/SumoLogic/sumologic-kubernetes-fluentd/tree/v1.12.2-sumo-4/fluent-plugin-protobuf -[telegrafreceiver]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/receiver/telegrafreceiver +[telegrafreceiver]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/receiver/telegrafreceiver ### fluent-plugin-prometheus-format @@ -162,17 +162,17 @@ Sanitized pod name is name portion of the pod. Please consider following example [kube_daemonset]: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/ [kube_deployment]: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/ [kube_statefulset]: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset -[otelcol_annotations]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/processor/sourceprocessor#pod-annotations +[otelcol_annotations]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/processor/sourceprocessor#pod-annotations [otelcol_annotations_exclude]: https://github.com/SumoLogic/sumologic-kubernetes-fluentd/blob/v1.12.2-sumo-4/fluent-plugin-kubernetes-sumologic/lib/fluent/plugin/filter_kubernetes_sumologic.rb#L171-L183 -[otelcol_source_templates]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/exporter/sumologicexporter#source-templates -[otelcol_undefined]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.47.0-sumo-0/pkg/processor/sourceprocessor/attribute_filler.go#L113-L123 -[source_containers]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/processor/sourceprocessor#container-level-pod-annotations -[source_filtering]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/processor/sourceprocessor#filtering-section -[source_keys]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/processor/sourceprocessor#keys-section -[source_processor]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/processor/sourceprocessor#config -[source_processor_source_templates]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/processor/sourceprocessor#name-translation-and-template-keys +[otelcol_source_templates]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/exporter/sumologicexporter#source-templates +[otelcol_undefined]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.50.0-sumo-0/pkg/processor/sourceprocessor/attribute_filler.go#L113-L123 +[source_containers]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/processor/sourceprocessor#container-level-pod-annotations +[source_filtering]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/processor/sourceprocessor#filtering-section +[source_keys]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/processor/sourceprocessor#keys-section +[source_processor]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/processor/sourceprocessor#config +[source_processor_source_templates]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/processor/sourceprocessor#name-translation-and-template-keys [sumo_metadata]: https://github.com/SumoLogic/sumologic-kubernetes-fluentd/tree/v1.12.2-sumo-4/fluent-plugin-kubernetes-sumologic#fluent-plugin-kubernetes-sumologic -[sumologic_exporter]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.47.0-sumo-0/pkg/exporter/sumologicexporter/README.md#sumo-logic-exporter +[sumologic_exporter]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.50.0-sumo-0/pkg/exporter/sumologicexporter/README.md#sumo-logic-exporter ### fluent-plugin-kubernetes-metadata-filter @@ -190,8 +190,8 @@ Sanitized pod name is name portion of the pod. Please consider following example | `cache_ttl` | N/A | [fluent_plugin_k8s_metadata]: https://github.com/SumoLogic/sumologic-kubernetes-fluentd/tree/v1.12.2-sumo-4/fluent-plugin-kubernetes-metadata-filter#configuration -[k8sprocessor]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.47.0-sumo-0/pkg/processor/k8sprocessor -[k8sprocessor_field_extract]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/processor/k8sprocessor#field-extract-config +[k8sprocessor]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.50.0-sumo-0/pkg/processor/k8sprocessor +[k8sprocessor_field_extract]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/processor/k8sprocessor#field-extract-config [kubeconfig_auth_type]: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/v0.47.0/internal/k8sconfig/config.go#L53-L60 ### fluent-plugin-enhance-k8s-metadata @@ -209,7 +209,7 @@ Sanitized pod name is name portion of the pod. Please consider following example | `data_type` | N/A | [fluent_plugin_enhance_k8s_metadata]: https://github.com/SumoLogic/sumologic-kubernetes-fluentd/tree/v1.12.2-sumo-4/fluent-plugin-enhance-k8s-metadata#configuration -[pod_association]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.47.0-sumo-0/pkg/processor/k8sprocessor/doc.go#L17-L46 +[pod_association]: https://github.com/SumoLogic/sumologic-otel-collector/blob/v0.50.0-sumo-0/pkg/processor/k8sprocessor/doc.go#L17-L46 ### fluent-plugin-events @@ -358,7 +358,7 @@ Events are not supported by `Opentelemetry Collector` [filter_processor]: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/v0.47.0/processor/filterprocessor#filter-processor [readme]: ../helm/sumologic/README.md -[source_containers]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.47.0-sumo-0/pkg/processor/sourceprocessor#container-level-pod-annotations +[source_containers]: https://github.com/SumoLogic/sumologic-otel-collector/tree/v0.50.0-sumo-0/pkg/processor/sourceprocessor#container-level-pod-annotations [persistent_queue]: https://github.com/open-telemetry/opentelemetry-collector/tree/release/v0.37.x/exporter/exporterhelper#persistent-queue [file_storage_extension]: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/release/v0.37.x/extension/storage/filestorage diff --git a/deploy/helm/sumologic/values.yaml b/deploy/helm/sumologic/values.yaml index 0ef84bcad5..9d0860811d 100644 --- a/deploy/helm/sumologic/values.yaml +++ b/deploy/helm/sumologic/values.yaml @@ -3543,10 +3543,6 @@ otelcol: source_name: "exporters.sumologic.source_name.replace" ## Desired host name. Useful if you want to override the source host configured for the source. source_host: "%{host.name}" - ## List of regexes for attributes which should be send as metadata - metadata_attributes: - - k8s.* - - host.name ## Timeout for every attempt to send data to Sumo Logic backend. Maximum connection timeout is 55s. timeout: 5s retry_on_failure: @@ -3585,7 +3581,7 @@ metadata: ## Configure image for Opentelemetry Collector (for logs and metrics) image: repository: public.ecr.aws/sumologic/sumologic-otel-collector - tag: 0.47.0-sumo-0 + tag: 0.50.0-sumo-0 pullPolicy: IfNotPresent securityContext: @@ -3693,11 +3689,6 @@ metadata: max_request_body_size: 16_777_216 # 16 MB before compression ## set timeout to 30s due to big requests timeout: 30s - metadata_attributes: - - k8s.* - - pod_labels_.* - - namespace_labels_.* - routing_atttribute_to_drop: http_listener_v2_path sumologic/apiserver: metric_format: prometheus endpoint: ${SUMO_ENDPOINT_APISERVER_METRICS_SOURCE} @@ -3709,11 +3700,6 @@ metadata: max_request_body_size: 16_777_216 # 16 MB before compression ## set timeout to 30s due to big requests timeout: 30s - metadata_attributes: - - k8s.* - - pod_labels_.* - - namespace_labels_.* - routing_atttribute_to_drop: http_listener_v2_path sumologic/control_plane: metric_format: prometheus endpoint: ${SUMO_ENDPOINT_CONTROL_PLANE_METRICS_SOURCE} @@ -3725,11 +3711,6 @@ metadata: max_request_body_size: 16_777_216 # 16 MB before compression ## set timeout to 30s due to big requests timeout: 30s - metadata_attributes: - - k8s.* - - pod_labels_.* - - namespace_labels_.* - routing_atttribute_to_drop: http_listener_v2_path sumologic/controller: metric_format: prometheus endpoint: ${SUMO_ENDPOINT_CONTROLLER_METRICS_SOURCE} @@ -3741,11 +3722,6 @@ metadata: max_request_body_size: 16_777_216 # 16 MB before compression ## set timeout to 30s due to big requests timeout: 30s - metadata_attributes: - - k8s.* - - pod_labels_.* - - namespace_labels_.* - routing_atttribute_to_drop: http_listener_v2_path sumologic/kubelet: metric_format: prometheus endpoint: ${SUMO_ENDPOINT_KUBELET_METRICS_SOURCE} @@ -3757,11 +3733,6 @@ metadata: max_request_body_size: 16_777_216 # 16 MB before compression ## set timeout to 30s due to big requests timeout: 30s - metadata_attributes: - - k8s.* - - pod_labels_.* - - namespace_labels_.* - routing_atttribute_to_drop: http_listener_v2_path sumologic/node: metric_format: prometheus endpoint: ${SUMO_ENDPOINT_NODE_METRICS_SOURCE} @@ -3773,11 +3744,6 @@ metadata: max_request_body_size: 16_777_216 # 16 MB before compression ## set timeout to 30s due to big requests timeout: 30s - metadata_attributes: - - k8s.* - - pod_labels_.* - - namespace_labels_.* - routing_atttribute_to_drop: http_listener_v2_path sumologic/scheduler: metric_format: prometheus endpoint: ${SUMO_ENDPOINT_SCHEDULER_METRICS_SOURCE} @@ -3789,11 +3755,6 @@ metadata: max_request_body_size: 16_777_216 # 16 MB before compression ## set timeout to 30s due to big requests timeout: 30s - metadata_attributes: - - k8s.* - - pod_labels_.* - - namespace_labels_.* - routing_atttribute_to_drop: http_listener_v2_path sumologic/state: metric_format: prometheus endpoint: ${SUMO_ENDPOINT_STATE_METRICS_SOURCE} @@ -3805,11 +3766,6 @@ metadata: max_request_body_size: 16_777_216 # 16 MB before compression ## set timeout to 30s due to big requests timeout: 30s - metadata_attributes: - - k8s.* - - pod_labels_.* - - namespace_labels_.* - routing_atttribute_to_drop: http_listener_v2_path processors: ## Configuration for Metrics Transform Processor ## ref: https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/release/v0.37.x/processor/metricstransformprocessor @@ -3870,6 +3826,7 @@ metadata: ## NOTE: add a feature to routingprocessor to drop the routing ## attribute to prevent it being sent to the exporters. from_attribute: http_listener_v2_path + drop_resource_routing_attribute: true attribute_source: resource default_exporters: - sumologic/default @@ -4105,12 +4062,6 @@ metadata: sending_queue: enabled: true persistent_storage_enabled: '{{ .Values.metadata.persistence.enabled }}' - metadata_attributes: - ## Attributes to be send as fields - - _collector - - k8s.* - - pod_labels_.* - - namespace_labels_.* sumologic/systemd: log_format: json json_logs: @@ -4125,8 +4076,6 @@ metadata: sending_queue: enabled: true persistent_storage_enabled: '{{ .Values.metadata.persistence.enabled }}' - metadata_attributes: - - _collector processors: ## Common processors @@ -4192,12 +4141,17 @@ metadata: - action: upsert key: k8s.pod.hostname from_attribute: k8s.node.name + resource/drop_annotations: + attributes: + - pattern: ^pod_annotations_.* + action: delete groupbyattrs/containers: keys: - k8s.container.id - k8s.container.name - k8s.namespace.name - k8s.pod.name + - _collector k8s_tagger: ## Has to be false to enrich metadata passthrough: false @@ -4277,13 +4231,67 @@ metadata: record_attributes: - key: _SYSTEMD_UNIT value: kubelet.service + filter/exclude_systemd_syslog: + logs: + exclude: + match_type: regexp + record_attributes: + - key: SYSLOG_FACILITY + value: '{{ .Values.fluentd.logs.systemd.excludeFacilityRegex | default "$^" | quote }}' + filter/exclude_systemd_hostname: + logs: + exclude: + match_type: regexp + record_attributes: + - key: _HOSTNAME + value: '{{ .Values.fluentd.logs.systemd.excludeHostRegex | default "$^" | quote }}' + filter/exclude_systemd_priority: + logs: + exclude: + match_type: regexp + record_attributes: + - key: PRIORITY + value: '{{ .Values.fluentd.logs.systemd.excludePriorityRegex | default "$^" | quote }}' + filter/exclude_systemd_unit: + logs: + exclude: + match_type: regexp + record_attributes: + - key: _SYSTEMD_UNIT + value: '{{ .Values.fluentd.logs.systemd.excludeUnitRegex | default "$^" | quote }}' + filter/exclude_kubelet_syslog: + logs: + exclude: + match_type: regexp + record_attributes: + - key: SYSLOG_FACILITY + value: '{{ .Values.fluentd.logs.kubelet.excludeFacilityRegex | default "$^" | quote }}' + filter/exclude_kubelet_hostname: + logs: + exclude: + match_type: regexp + record_attributes: + - key: _HOSTNAME + value: '{{ .Values.fluentd.logs.kubelet.excludeHostRegex | default "$^" | quote }}' + filter/exclude_kubelet_priority: + logs: + exclude: + match_type: regexp + record_attributes: + - key: PRIORITY + value: '{{ .Values.fluentd.logs.kubelet.excludePriorityRegex | default "$^" | quote }}' + filter/exclude_kubelet_unit: + logs: + exclude: + match_type: regexp + record_attributes: + - key: _SYSTEMD_UNIT + value: '{{ .Values.fluentd.logs.kubelet.excludeUnitRegex | default "$^" | quote }}' + groupbyattrs/systemd: keys: - - _SYSTEMD_UNIT - - _HOSTNAME - - SYSLOG_FACILITY - - PRIORITY - _sourceName + - _collector source/systemd: collector: '{{ .Values.sumologic.collectorName | default .Values.sumologic.clusterName | quote }}' source_host: "%{_HOSTNAME}" @@ -4291,11 +4299,6 @@ metadata: source_category: '{{ .Values.fluentd.logs.systemd.sourceCategory | quote }}' source_category_prefix: '{{ .Values.fluentd.logs.systemd.sourceCategoryPrefix | quote }}' source_category_replace_dash: '{{ .Values.fluentd.logs.systemd.sourceCategoryReplaceDash | quote }}' - exclude: - SYSLOG_FACILITY: '{{ .Values.fluentd.logs.systemd.excludeFacilityRegex | quote }}' - _HOSTNAME: '{{ .Values.fluentd.logs.systemd.excludeHostRegex | quote }}' - PRIORITY: '{{ .Values.fluentd.logs.systemd.excludePriorityRegex | quote }}' - _SYSTEMD_UNIT: '{{ .Values.fluentd.logs.systemd.excludeUnitRegex | quote }}' ## kubelet related processors filter/include_kubelet: @@ -4312,11 +4315,6 @@ metadata: source_category: '{{ .Values.fluentd.logs.kubelet.sourceCategory | quote }}' source_category_prefix: '{{ .Values.fluentd.logs.kubelet.sourceCategoryPrefix | quote }}' source_category_replace_dash: '{{ .Values.fluentd.logs.kubelet.sourceCategoryReplaceDash | quote }}' - exclude: - SYSLOG_FACILITY: '{{ .Values.fluentd.logs.kubelet.excludeFacilityRegex | quote }}' - _HOSTNAME: '{{ .Values.fluentd.logs.kubelet.excludeHostRegex | quote }}' - PRIORITY: '{{ .Values.fluentd.logs.kubelet.excludePriorityRegex | quote }}' - _SYSTEMD_UNIT: '{{ .Values.fluentd.logs.kubelet.excludeUnitRegex | quote }}' service: telemetry: @@ -4337,6 +4335,7 @@ metadata: - groupbyattrs/containers - k8s_tagger - source/containers + - resource/drop_annotations - attributes/remove_fluent_tag - resource/containers_copy_node_to_host - batch @@ -4356,6 +4355,7 @@ metadata: # - groupbyattrs/containers # - k8s_tagger # - source/containers + # - resource/drop_annotations # - resource/containers_copy_node_to_host # - batch # exporters: @@ -4368,6 +4368,10 @@ metadata: - filter/include_fluent_tag_host - filter/include_systemd - filter/exclude_kubelet + - filter/exclude_systemd_syslog + - filter/exclude_systemd_hostname + - filter/exclude_systemd_priority + - filter/exclude_systemd_unit - groupbyattrs/systemd - source/systemd - attributes/remove_fluent_tag @@ -4381,6 +4385,10 @@ metadata: - memory_limiter - filter/include_fluent_tag_host - filter/include_kubelet + - filter/exclude_kubelet_syslog + - filter/exclude_kubelet_hostname + - filter/exclude_kubelet_priority + - filter/exclude_kubelet_unit - groupbyattrs/systemd - source/kubelet - attributes/remove_fluent_tag @@ -4484,7 +4492,7 @@ otelgateway: podAnnotations: {} image: repository: public.ecr.aws/sumologic/sumologic-otel-collector - tag: 0.47.0-sumo-0 + tag: 0.50.0-sumo-0 pullPolicy: IfNotPresent livenessProbe: periodSeconds: 15 diff --git a/tests/helm/metadata_logs_otc/static/basic.output.yaml b/tests/helm/metadata_logs_otc/static/basic.output.yaml index ea4d56f742..840a8b9f92 100644 --- a/tests/helm/metadata_logs_otc/static/basic.output.yaml +++ b/tests/helm/metadata_logs_otc/static/basic.output.yaml @@ -19,11 +19,6 @@ data: add_timestamp: true timestamp_key: timestamp log_format: json - metadata_attributes: - - _collector - - k8s.* - - pod_labels_.* - - namespace_labels_.* sending_queue: enabled: true persistent_storage_enabled: true @@ -36,8 +31,6 @@ data: add_timestamp: true timestamp_key: timestamp log_format: json - metadata_attributes: - - _collector sending_queue: enabled: true persistent_storage_enabled: true @@ -95,6 +88,62 @@ data: record_attributes: - key: _SYSTEMD_UNIT value: kubelet.service + filter/exclude_kubelet_hostname: + logs: + exclude: + match_type: regexp + record_attributes: + - key: _HOSTNAME + value: "$^" + filter/exclude_kubelet_priority: + logs: + exclude: + match_type: regexp + record_attributes: + - key: PRIORITY + value: "$^" + filter/exclude_kubelet_syslog: + logs: + exclude: + match_type: regexp + record_attributes: + - key: SYSLOG_FACILITY + value: "$^" + filter/exclude_kubelet_unit: + logs: + exclude: + match_type: regexp + record_attributes: + - key: _SYSTEMD_UNIT + value: "$^" + filter/exclude_systemd_hostname: + logs: + exclude: + match_type: regexp + record_attributes: + - key: _HOSTNAME + value: "$^" + filter/exclude_systemd_priority: + logs: + exclude: + match_type: regexp + record_attributes: + - key: PRIORITY + value: "$^" + filter/exclude_systemd_syslog: + logs: + exclude: + match_type: regexp + record_attributes: + - key: SYSLOG_FACILITY + value: "$^" + filter/exclude_systemd_unit: + logs: + exclude: + match_type: regexp + record_attributes: + - key: _SYSTEMD_UNIT + value: "$^" filter/include_fluent_tag_containers: logs: include: @@ -129,13 +178,11 @@ data: - k8s.container.name - k8s.namespace.name - k8s.pod.name + - _collector groupbyattrs/systemd: keys: - - _SYSTEMD_UNIT - - _HOSTNAME - - SYSLOG_FACILITY - - PRIORITY - _sourceName + - _collector k8s_tagger: extract: annotations: @@ -175,6 +222,10 @@ data: - action: upsert from_attribute: k8s.node.name key: k8s.pod.hostname + resource/drop_annotations: + attributes: + - action: delete + pattern: ^pod_annotations_.* source/containers: annotation_prefix: pod_annotations_ collector: "kubernetes" @@ -196,11 +247,6 @@ data: source_name: '%{k8s.namespace.name}.%{k8s.pod.name}.%{k8s.container.name}' source/kubelet: collector: "kubernetes" - exclude: - _HOSTNAME: "" - _SYSTEMD_UNIT: "" - PRIORITY: "" - SYSLOG_FACILITY: "" source_category: "kubelet" source_category_prefix: "kubernetes/" source_category_replace_dash: "/" @@ -208,11 +254,6 @@ data: source_name: "k8s_kubelet" source/systemd: collector: "kubernetes" - exclude: - _HOSTNAME: "" - _SYSTEMD_UNIT: "" - PRIORITY: "" - SYSLOG_FACILITY: "" source_category: "system" source_category_prefix: "kubernetes/" source_category_replace_dash: "/" @@ -241,6 +282,7 @@ data: - groupbyattrs/containers - k8s_tagger - source/containers + - resource/drop_annotations - attributes/remove_fluent_tag - resource/containers_copy_node_to_host - batch @@ -253,6 +295,10 @@ data: - memory_limiter - filter/include_fluent_tag_host - filter/include_kubelet + - filter/exclude_kubelet_syslog + - filter/exclude_kubelet_hostname + - filter/exclude_kubelet_priority + - filter/exclude_kubelet_unit - groupbyattrs/systemd - source/kubelet - attributes/remove_fluent_tag @@ -267,6 +313,10 @@ data: - filter/include_fluent_tag_host - filter/include_systemd - filter/exclude_kubelet + - filter/exclude_systemd_syslog + - filter/exclude_systemd_hostname + - filter/exclude_systemd_priority + - filter/exclude_systemd_unit - groupbyattrs/systemd - source/systemd - attributes/remove_fluent_tag diff --git a/tests/helm/metadata_logs_otc/static/templates.output.yaml b/tests/helm/metadata_logs_otc/static/templates.output.yaml index fb9082caa8..cc7efd30e6 100644 --- a/tests/helm/metadata_logs_otc/static/templates.output.yaml +++ b/tests/helm/metadata_logs_otc/static/templates.output.yaml @@ -19,11 +19,6 @@ data: add_timestamp: true timestamp_key: timestamp log_format: json - metadata_attributes: - - _collector - - k8s.* - - pod_labels_.* - - namespace_labels_.* sending_queue: enabled: true persistent_storage_enabled: true @@ -36,8 +31,6 @@ data: add_timestamp: true timestamp_key: timestamp log_format: json - metadata_attributes: - - _collector sending_queue: enabled: true persistent_storage_enabled: true @@ -95,6 +88,62 @@ data: record_attributes: - key: _SYSTEMD_UNIT value: kubelet.service + filter/exclude_kubelet_hostname: + logs: + exclude: + match_type: regexp + record_attributes: + - key: _HOSTNAME + value: "my_kubelet_excludeHostRegex" + filter/exclude_kubelet_priority: + logs: + exclude: + match_type: regexp + record_attributes: + - key: PRIORITY + value: "my_kubelet_excludePriorityRegex" + filter/exclude_kubelet_syslog: + logs: + exclude: + match_type: regexp + record_attributes: + - key: SYSLOG_FACILITY + value: "my_kubelet_excludeFacilityRegex" + filter/exclude_kubelet_unit: + logs: + exclude: + match_type: regexp + record_attributes: + - key: _SYSTEMD_UNIT + value: "my_kubelet_excludeUnitRegex" + filter/exclude_systemd_hostname: + logs: + exclude: + match_type: regexp + record_attributes: + - key: _HOSTNAME + value: "my_systemd_excludeHostRegex" + filter/exclude_systemd_priority: + logs: + exclude: + match_type: regexp + record_attributes: + - key: PRIORITY + value: "my_systemd_excludePriorityRegex" + filter/exclude_systemd_syslog: + logs: + exclude: + match_type: regexp + record_attributes: + - key: SYSLOG_FACILITY + value: "my_systemd_excludeFacilityRegex" + filter/exclude_systemd_unit: + logs: + exclude: + match_type: regexp + record_attributes: + - key: _SYSTEMD_UNIT + value: "my_systemd_excludeUnitRegex" filter/include_fluent_tag_containers: logs: include: @@ -129,13 +178,11 @@ data: - k8s.container.name - k8s.namespace.name - k8s.pod.name + - _collector groupbyattrs/systemd: keys: - - _SYSTEMD_UNIT - - _HOSTNAME - - SYSLOG_FACILITY - - PRIORITY - _sourceName + - _collector k8s_tagger: extract: annotations: @@ -175,6 +222,10 @@ data: - action: upsert from_attribute: k8s.node.name key: k8s.pod.hostname + resource/drop_annotations: + attributes: + - action: delete + pattern: ^pod_annotations_.* source/containers: annotation_prefix: pod_annotations_ collector: "my_collectorName" @@ -196,11 +247,6 @@ data: source_name: '%{k8s.namespace.name}.%{k8s.pod.name}.%{k8s.container.name}' source/kubelet: collector: "my_collectorName" - exclude: - _HOSTNAME: "my_kubelet_excludeHostRegex" - _SYSTEMD_UNIT: "my_kubelet_excludeUnitRegex" - PRIORITY: "my_kubelet_excludePriorityRegex" - SYSLOG_FACILITY: "my_kubelet_excludeFacilityRegex" source_category: "kubelet" source_category_prefix: "my_kubelet_sourceCategoryPrefix" source_category_replace_dash: "my_kubelet_sourceCategoryReplaceDash" @@ -208,11 +254,6 @@ data: source_name: "k8s_kubelet" source/systemd: collector: "my_collectorName" - exclude: - _HOSTNAME: "my_systemd_excludeHostRegex" - _SYSTEMD_UNIT: "my_systemd_excludeUnitRegex" - PRIORITY: "my_systemd_excludePriorityRegex" - SYSLOG_FACILITY: "my_systemd_excludeFacilityRegex" source_category: "system" source_category_prefix: "my_systemd_sourceCategoryPrefix" source_category_replace_dash: "my_systemd_sourceCategoryReplaceDash" @@ -241,6 +282,7 @@ data: - groupbyattrs/containers - k8s_tagger - source/containers + - resource/drop_annotations - attributes/remove_fluent_tag - resource/containers_copy_node_to_host - batch @@ -253,6 +295,10 @@ data: - memory_limiter - filter/include_fluent_tag_host - filter/include_kubelet + - filter/exclude_kubelet_syslog + - filter/exclude_kubelet_hostname + - filter/exclude_kubelet_priority + - filter/exclude_kubelet_unit - groupbyattrs/systemd - source/kubelet - attributes/remove_fluent_tag @@ -267,6 +313,10 @@ data: - filter/include_fluent_tag_host - filter/include_systemd - filter/exclude_kubelet + - filter/exclude_systemd_syslog + - filter/exclude_systemd_hostname + - filter/exclude_systemd_priority + - filter/exclude_systemd_unit - groupbyattrs/systemd - source/systemd - attributes/remove_fluent_tag diff --git a/tests/helm/metadata_logs_otc_statefulset/static/basic.output.yaml b/tests/helm/metadata_logs_otc_statefulset/static/basic.output.yaml index 87b857b653..db7e0bc168 100644 --- a/tests/helm/metadata_logs_otc_statefulset/static/basic.output.yaml +++ b/tests/helm/metadata_logs_otc_statefulset/static/basic.output.yaml @@ -65,7 +65,7 @@ spec: priorityClassName: "prio" containers: - name: otelcol - image: public.ecr.aws/sumologic/sumologic-otel-collector:0.47.0-sumo-0 + image: public.ecr.aws/sumologic/sumologic-otel-collector:0.50.0-sumo-0 imagePullPolicy: IfNotPresent args: - --config=/etc/otel/config.yaml diff --git a/tests/helm/metadata_metrics_otc_statefulset/static/basic.output.yaml b/tests/helm/metadata_metrics_otc_statefulset/static/basic.output.yaml index 4529a63d2b..7dbb045225 100644 --- a/tests/helm/metadata_metrics_otc_statefulset/static/basic.output.yaml +++ b/tests/helm/metadata_metrics_otc_statefulset/static/basic.output.yaml @@ -65,7 +65,7 @@ spec: priorityClassName: "prio" containers: - name: otelcol - image: public.ecr.aws/sumologic/sumologic-otel-collector:0.47.0-sumo-0 + image: public.ecr.aws/sumologic/sumologic-otel-collector:0.50.0-sumo-0 imagePullPolicy: IfNotPresent args: - --config=/etc/otel/config.yaml diff --git a/tests/helm/tracing/static/collection-monitoring-false.output.yaml b/tests/helm/tracing/static/collection-monitoring-false.output.yaml index 36c36235fe..1c7c95c01f 100644 --- a/tests/helm/tracing/static/collection-monitoring-false.output.yaml +++ b/tests/helm/tracing/static/collection-monitoring-false.output.yaml @@ -20,9 +20,6 @@ data: endpoint: ${SUMO_ENDPOINT_DEFAULT_METRICS_SOURCE} log_format: text max_request_body_size: 1048576 - metadata_attributes: - - k8s.* - - host.name metric_format: prometheus retry_on_failure: enabled: true diff --git a/tests/helm/tracing/static/simple.output.yaml b/tests/helm/tracing/static/simple.output.yaml index b3821b6c88..47ca46dc3d 100644 --- a/tests/helm/tracing/static/simple.output.yaml +++ b/tests/helm/tracing/static/simple.output.yaml @@ -20,9 +20,6 @@ data: endpoint: ${SUMO_ENDPOINT_DEFAULT_METRICS_SOURCE} log_format: text max_request_body_size: 1048576 - metadata_attributes: - - k8s.* - - host.name metric_format: prometheus retry_on_failure: enabled: true diff --git a/vagrant/values.yaml b/vagrant/values.yaml index d4b62c6da5..391403b045 100644 --- a/vagrant/values.yaml +++ b/vagrant/values.yaml @@ -186,9 +186,6 @@ metadata: - action: extract key: fluent.tag pattern: ^containers\.var\.log\.pods\.(?P[^_]+)_(?P[^_]+)_(?P[a-f0-9\-]{36})\.(?P[^\._]+)\.(?P\d+)\.log$ - - action: insert - key: k8s.pod.uid - from_attribute: k8s_uid - action: delete key: k8s_uid - action: delete