example: add new opentelemetry filter to filter tracetest traces (#3422)

* examples: add example on how to filter non-tracetest traces * fix: export tracetest traces to tracetest * docs: update instructions on how to sample tracetest spans * docs: mention tail sampling * Apply suggestions from code review Co-authored-by: Julianne Fermi <julianne@kubeshop.io> --------- Co-authored-by: Julianne Fermi <julianne@kubeshop.io>
kubeshop · Dec 4, 2023 · 8ea7a6c · 8ea7a6c
1 parent a4c4cda
commit 8ea7a6c
Show file tree

Hide file tree

Showing 6 changed files with 174 additions and 16 deletions.
diff --git a/docs/docs/configuration/sampling-tracetest-spans.mdx b/docs/docs/configuration/sampling-tracetest-spans.mdx
@@ -16,9 +16,6 @@ your test spans not sampled by your probabilistic sampler. There are a couple of
 
 ## Add a Separate Pipeline for Tracetest in your OpenTelemetry Collector
 
-> :warning: Note: This requires the [OpenTelemetry collector contrib](https://hub.docker.com/r/otel/opentelemetry-collector-contrib) instead of the core release
-> of the collector
-
 Your existing OpenTelemetry Collector already receives traces from your applications and sends them to your datastore and you have a set of processors configured to ensure the quality of the traces your datastore is receiving. It probably looks like this one:
 
 ```yaml
@@ -59,22 +56,20 @@ receivers:
       http:
 
 processors:
+  batch:
+
   probabilistic_sampler:
     hash_seed: 22
     sampling_percentage: 5.0
 
-  batch:
-
-  # Filters spans that have the tracestate `tracetest=true` in their context. This value
-  # is injected by Tracetest when triggering the test
-  #
-  # Note: this requires the `collector-contrib` version of the collector
-  tail_sampling:
-    decision_wait: 5s
-    policies:
-      - name: tracetest-spans
-        type: trace_state
-        trace_state: { key: tracetest, values: ["true"] }
+  # If this configuration fails on your collector, make sure to update it to a newer version.
+  # This is the recommended way of filtering spans based on the `trace_state`. It's faster and less
+  # resource intensive than using a `tail_sampling` approach.
+  filter/tracetest:
+    error_mode: ignore
+    traces:
+      span:
+        - 'trace_state["tracetest"] != "true"'
 
 exporters:
   otlp/jaeger:
@@ -92,10 +87,26 @@ service:
   pipelines:
     traces/tracetest:
       receivers: [otlp]
-      processors: [tail_sampling, batch]
+      processors: [filter/tracetest, batch]
       exporters: [otlp/jaeger]
 ```
 
+### Tail Sampling Approach
+Before December 2023, we were suggesting people to use tail sampling to filter the traces generated by Tracetest.
+However, the new `filter` capabilities are better for performance than tail sampling as it
+requires less memory to decide if a trace should be sampled or not. If you are not concerned with memory usage and still want to use the
+tail sampling approach, this is the processor you can use:
+
+```yaml
+processors:
+  tail_sampling:
+    decision_wait: 5s
+    policies:
+      - name: tracetest-spans
+        type: trace_state
+        trace_state: { key: tracetest, values: ["true"] }
+```
+
 With this configuration, you will still get 5% of all your traces, but you will also ensure that all your test traces are collected and sent to
 Jaeger.
 

diff --git a/examples/collector-filtering/collector.config.yaml b/examples/collector-filtering/collector.config.yaml
@@ -0,0 +1,31 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+      http:
+
+processors:
+  batch:
+    timeout: 100ms
+
+  filter/tracetest:
+    error_mode: ignore
+    traces:
+      span:
+        - 'trace_state["tracetest"] != "true"'
+
+exporters:
+  logging:
+    loglevel: debug
+
+  otlp/1:
+    endpoint: ${TRACETEST_ENDPOINT}
+    tls:
+      insecure: true
+
+service:
+  pipelines:
+    traces/1:
+      receivers: [otlp]
+      processors: [filter/tracetest, batch]
+      exporters: [otlp/1]
diff --git a/examples/collector-filtering/docker-compose.yml b/examples/collector-filtering/docker-compose.yml
@@ -0,0 +1,54 @@
+version: '3'
+services:
+
+    tracetest:
+        image: kubeshop/tracetest:${TAG:-latest}
+        # uncommentig this line breaks the portability of this file, that is the base for the installer
+        # platform: linux/amd64
+        volumes:
+            - type: bind
+              source: ./tracetest-config.yaml
+              target: /app/tracetest.yaml
+            - type: bind
+              source: ./tracetest-provision.yaml
+              target: /app/provision.yaml
+        command: --provisioning-file /app/provision.yaml
+        ports:
+            - 11633:11633
+        extra_hosts:
+          - "host.docker.internal:host-gateway"
+        depends_on:
+            postgres:
+                condition: service_healthy
+            otel-collector:
+                condition: service_started
+        healthcheck:
+            test: ["CMD", "wget", "--spider", "localhost:11633"]
+            interval: 1s
+            timeout: 3s
+            retries: 60
+        environment:
+            TRACETEST_DEV: ${TRACETEST_DEV}
+
+    postgres:
+        image: postgres:14
+        environment:
+            POSTGRES_PASSWORD: postgres
+            POSTGRES_USER: postgres
+        healthcheck:
+            test: pg_isready -U "$$POSTGRES_USER" -d "$$POSTGRES_DB"
+            interval: 1s
+            timeout: 5s
+            retries: 60
+
+    otel-collector:
+        image: otel/opentelemetry-collector:0.90.1
+        command:
+            - "--config"
+            - "/otel-local-config.yaml"
+        volumes:
+            - ./collector.config.yaml:/otel-local-config.yaml
+        ports:
+          - 4317:4317
+        environment:
+            - TRACETEST_ENDPOINT=tracetest:4317
diff --git a/examples/collector-filtering/tests/list-tests.yaml b/examples/collector-filtering/tests/list-tests.yaml
@@ -0,0 +1,17 @@
+type: Test
+spec:
+  id: e9c6cff9-974d-4263-8a23-22f1e9f975aa
+  name: List all tracetest tests
+  description: List all existing tests from tracetest API
+  trigger:
+    type: http
+    httpRequest:
+      url: http://localhost:11633/api/tests
+      method: GET
+      headers:
+      - key: Content-Type
+        value: application/json
+  specs:
+  - selector: span[tracetest.span.type="http" name="GET /api/tests"]
+    assertions:
+    - attr:tracetest.selected_spans.count = 1
diff --git a/examples/collector-filtering/tracetest-config.yaml b/examples/collector-filtering/tracetest-config.yaml
@@ -0,0 +1,21 @@
+postgres:
+  host: postgres
+  user: postgres
+  password: postgres
+  port: 5432
+  dbname: postgres
+  params: sslmode=disable
+
+telemetry:
+  exporters:
+    collector:
+      serviceName: tracetest
+      sampling: 100 # 100%
+      exporter:
+        type: collector
+        collector:
+          endpoint: otel-collector:4317
+
+server:
+  telemetry:
+    exporter: collector
diff --git a/examples/collector-filtering/tracetest-provision.yaml b/examples/collector-filtering/tracetest-provision.yaml
@@ -0,0 +1,24 @@
+---
+type: PollingProfile
+spec:
+  name: Default
+  strategy: periodic
+  default: true
+  periodic:
+    retryDelay: 5s
+    timeout: 10m
+
+---
+type: DataStore
+spec:
+  name: OpenTelemetry Collector
+  type: otlp
+  default: true
+---
+type: TestRunner
+spec:
+  id: current
+  name: default
+  requiredGates:
+    - analyzer-score
+    - test-specs