OpenTelemetry filter processor

components/telemetry-operator/controller/tracepipeline/resources.go

@@ -61,7 +61,7 @@ func makeConfigMap(config Config, output v1alpha1.TracePipelineOutput) *corev1.C
 			"pipelines": map[string]any{
 				"traces": map[string]any{
 					"receivers":  []any{"opencensus", "otlp"},
-					"processors": []any{"memory_limiter", "k8sattributes", "resource", "batch"},
+					"processors": []any{"memory_limiter", "k8sattributes", "filter","resource", "batch"},
 					"exporters":  []any{outputType},
 				},
 			},
@@ -177,6 +177,7 @@ func makeProcessorsConfig() map[string]any {
 			"timeout":             "10s",
 			"send_batch_max_size": 512,
 		},
+		"filter": makeFilterProcessorConfig(),
 		"memory_limiter": map[string]any{
 			"check_interval":         "1s",
 			"limit_percentage":       75,
@@ -202,6 +203,24 @@ func makeProcessorsConfig() map[string]any {
 	}
 }
 
+func makeFilterProcessorConfig() map[string]any {
+	return map[string]any{
+		"traces": map[string]any{
+			"span": []any{
+				"(attributes[\"http.method\"] == \"POST\") and (attributes[\"component\"] == \"proxy\") and (attributes[\"OperationName\"] == \"Ingress\") and (resource.attributes[\"service.name\"] == \"jaeger.kyma-system\")",
+				"(attributes[\"http.method\"] == \"GET\") and (attributes[\"component\"] == \"proxy\") and (attributes[\"OperationName\"] == \"Egress\") and (resource.attributes[\"service.name\"] == \"grafana.kyma-system\")",
+				"(attributes[\"http.method\"] == \"GET\") and (attributes[\"component\"] == \"proxy\") and (attributes[\"OperationName\"] == \"Ingress\") and (IsMatch(attributes[\"http.url\"], \".+/metrics\") == true) and (resource.attributes[\"k8s.namespace.name\"] == \"kyma-system\")",
+				"(attributes[\"http.method\"] == \"GET\") and (attributes[\"component\"] == \"proxy\") and (attributes[\"OperationName\"] == \"Ingress\") and (IsMatch(attributes[\"http.url\"], \".+/healthz(/.*)?\") == true) and (resource.attributes[\"k8s.namespace.name\"] == \"kyma-system\")",
+				"(attributes[\"http.method\"] == \"GET\") and (attributes[\"component\"] == \"proxy\") and (attributes[\"OperationName\"] == \"Ingress\") and (attributes[\"user_agent\"] == \"vm_promscrape\")",
+				"(attributes[\"http.method\"] == \"POST\") and (attributes[\"component\"] == \"proxy\") and (attributes[\"OperationName\"] == \"Egress\") and (IsMatch(attributes[\"http.url\"], \"http(s)?:\\\\/\\\\/telemetry-otlp-traces\\\\.kyma-system(\\\\..*)?:(4318|4317).*\") == true)",
+				"(attributes[\"http.method\"] == \"POST\") and (attributes[\"component\"] == \"proxy\") and (attributes[\"OperationName\"] == \"Egress\") and (IsMatch(attributes[\"http.url\"], \"http(s)?:\\\\/\\\\/telemetry-trace-collector-internal\\\\.kyma-system(\\\\..*)?:(55678).*\") == true)",
+				"(attributes[\"http.method\"] == \"POST\") and (attributes[\"component\"] == \"proxy\") and (attributes[\"OperationName\"] == \"Ingress\") and (resource.attributes[\"service.name\"] == \"loki.kyma-system\")",
+				"(attributes[\"http.method\"] == \"POST\") and (attributes[\"component\"] == \"proxy\") and (attributes[\"OperationName\"] == \"Egress\") and (resource.attributes[\"service.name\"] == \"telemetry-fluent-bit.kyma-system\")",
+			},
+		},
+	}
+}
+
 func makeDeployment(config Config, configHash string) *appsv1.Deployment {
 	labels := makeDefaultLabels(config)
 	optional := true

components/telemetry-operator/controller/tracepipeline/resources_test.go

@@ -76,6 +76,15 @@ func TestMakeConfigMap(t *testing.T) {
 	require.True(t, strings.Contains(collectorConfig, expectedEndpoint), "Otel Collector config must contain OTLP endpoint")
 }
 
+func TestMakeConfigMapFilterProcessorConfig(t *testing.T) {
+	cm := makeConfigMap(config, tracePipeline)
+	collectorConfig := cm.Data[configMapKey]
+	filetConf := "filter:\n        traces:\n            span:"
+	filterExpressions := "- (attributes[\"http.method\"] == \"GET\") and (attributes[\"component\"] == \"proxy\") and (attributes[\"OperationName\"] == \"Egress\") and (resource.attributes[\"service.name\"] == \"jaeger.kyma-system\")\n                - (attributes[\"http.method\"] == \"GET\") and (attributes[\"component\"] == \"proxy\") and (attributes[\"OperationName\"] == \"Egress\") and (resource.attributes[\"service.name\"] == \"grafana.kyma-system\")\n                - (attributes[\"http.method\"] == \"GET\") and (attributes[\"component\"] == \"proxy\") and (attributes[\"OperationName\"] == \"Ingress\") and (IsMatch(attributes[\"http.url\"], \".+/metrics\") == true) and (resource.attributes[\"k8s.namespace.name\"] == \"kyma-system\")\n                - (attributes[\"http.method\"] == \"GET\") and (attributes[\"component\"] == \"proxy\") and (attributes[\"OperationName\"] == \"Ingress\") and (IsMatch(attributes[\"http.url\"], \".+/healthz(/.*)?\") == true) and (resource.attributes[\"k8s.namespace.name\"] == \"kyma-system\")\n                - (attributes[\"http.method\"] == \"GET\") and (attributes[\"component\"] == \"proxy\") and (attributes[\"OperationName\"] == \"Ingress\") and (attributes[\"user_agent\"] == \"vm_promscrape\")\n                - (attributes[\"http.method\"] == \"POST\") and (attributes[\"component\"] == \"proxy\") and (attributes[\"OperationName\"] == \"Egress\") and (IsMatch(attributes[\"http.url\"], \"http(s)?:\\\\/\\\\/telemetry-otlp-traces\\\\.kyma-system(\\\\..*)?:(4318|4317).*\") == true)\n                - (attributes[\"http.method\"] == \"POST\") and (attributes[\"component\"] == \"proxy\") and (attributes[\"OperationName\"] == \"Egress\") and (IsMatch(attributes[\"http.url\"], \"http(s)?:\\\\/\\\\/telemetry-trace-collector-internal\\\\.kyma-system(\\\\..*)?:(55678).*\") == true)\n                - (attributes[\"http.method\"] == \"POST\") and (attributes[\"component\"] == \"proxy\") and (attributes[\"OperationName\"] == \"Ingress\") and (resource.attributes[\"service.name\"] == \"loki.kyma-system\")\n                - (attributes[\"http.method\"] == \"POST\") and (attributes[\"component\"] == \"proxy\") and (attributes[\"OperationName\"] == \"Egress\") and (resource.attributes[\"service.name\"] == \"telemetry-fluent-bit.kyma-system\")"
+	require.True(t, strings.Contains(collectorConfig, filetConf), "Otel Collector configmap must contain filter config")
+	require.True(t, strings.Contains(collectorConfig, filterExpressions), "Otel Collector configmap must contain filter expression")
+}
+
 func TestMakeConfigMapTLSInsecureNoScheme(t *testing.T) {
 	cm := makeConfigMap(config, tracePipeline)
 

docs/01-overview/main-areas/telemetry/telemetry-03-traces.md

@@ -264,6 +264,12 @@ The trace collector setup is designed using the following assumptions:
 - An unavailability of a destination must be survived for 5 minutes without direct loss of trace data
 - An average span consists of 40 attributes with 64 character length
 
+The trace collector setup is configured filter out following traces and spans:
+- `/healtz` endpoint of any component deployed on `kyma-system` namespace
+- `/metrics` endpoint of any component deployed on `kyma-system namespace
+- All traces generated from Grafana and Jaeger deployments
+- All traces in reference to `fluent-bit` and `loki` communication 
+
 It results in the following limitations:
 ### Throughput
 The maximum throughput is 4200 span/sec ~= 15.000.000 spans/hour. If more data needs to be ingested, it can result in a refusal of more data.

resources/telemetry/values.yaml

@@ -13,7 +13,7 @@ global:
       version: "v20221122-200937b4"
     telemetry_otel_collector:
       name: "otel-collector"
-      version: "0.66.0-a80d981f"
+      version: "0.66.0-8bb1c644"
       directory: "tpi"
     fluent_bit:
       name: "fluent-bit"