diff --git a/docs/docs/configuration/sampling-tracetest-spans.mdx b/docs/docs/configuration/sampling-tracetest-spans.mdx index 7eedfd7b0b..a04c6f5797 100644 --- a/docs/docs/configuration/sampling-tracetest-spans.mdx +++ b/docs/docs/configuration/sampling-tracetest-spans.mdx @@ -16,9 +16,6 @@ your test spans not sampled by your probabilistic sampler. There are a couple of ## Add a Separate Pipeline for Tracetest in your OpenTelemetry Collector -> :warning: Note: This requires the [OpenTelemetry collector contrib](https://hub.docker.com/r/otel/opentelemetry-collector-contrib) instead of the core release -> of the collector - Your existing OpenTelemetry Collector already receives traces from your applications and sends them to your datastore and you have a set of processors configured to ensure the quality of the traces your datastore is receiving. It probably looks like this one: ```yaml @@ -59,22 +56,20 @@ receivers: http: processors: + batch: + probabilistic_sampler: hash_seed: 22 sampling_percentage: 5.0 - batch: - - # Filters spans that have the tracestate `tracetest=true` in their context. This value - # is injected by Tracetest when triggering the test - # - # Note: this requires the `collector-contrib` version of the collector - tail_sampling: - decision_wait: 5s - policies: - - name: tracetest-spans - type: trace_state - trace_state: { key: tracetest, values: ["true"] } + # If this configuration fails on your collector, make sure to update it to a newer version. + # This is the recommended way of filtering spans based on the `trace_state`. It's faster and less + # resource intensive than using a `tail_sampling` approach. + filter/tracetest: + error_mode: ignore + traces: + span: + - 'trace_state["tracetest"] != "true"' exporters: otlp/jaeger: @@ -92,10 +87,26 @@ service: pipelines: traces/tracetest: receivers: [otlp] - processors: [tail_sampling, batch] + processors: [filter/tracetest, batch] exporters: [otlp/jaeger] ``` +### Tail Sampling Approach +Before December 2023, we were suggesting people to use tail sampling to filter the traces generated by Tracetest. +However, the new `filter` capabilities are better for performance than tail sampling as it +requires less memory to decide if a trace should be sampled or not. If you are not concerned with memory usage and still want to use the +tail sampling approach, this is the processor you can use: + +```yaml +processors: + tail_sampling: + decision_wait: 5s + policies: + - name: tracetest-spans + type: trace_state + trace_state: { key: tracetest, values: ["true"] } +``` + With this configuration, you will still get 5% of all your traces, but you will also ensure that all your test traces are collected and sent to Jaeger. diff --git a/examples/collector-filtering/collector.config.yaml b/examples/collector-filtering/collector.config.yaml new file mode 100644 index 0000000000..618668139c --- /dev/null +++ b/examples/collector-filtering/collector.config.yaml @@ -0,0 +1,31 @@ +receivers: + otlp: + protocols: + grpc: + http: + +processors: + batch: + timeout: 100ms + + filter/tracetest: + error_mode: ignore + traces: + span: + - 'trace_state["tracetest"] != "true"' + +exporters: + logging: + loglevel: debug + + otlp/1: + endpoint: ${TRACETEST_ENDPOINT} + tls: + insecure: true + +service: + pipelines: + traces/1: + receivers: [otlp] + processors: [filter/tracetest, batch] + exporters: [otlp/1] diff --git a/examples/collector-filtering/docker-compose.yml b/examples/collector-filtering/docker-compose.yml new file mode 100644 index 0000000000..77c774d4ed --- /dev/null +++ b/examples/collector-filtering/docker-compose.yml @@ -0,0 +1,54 @@ +version: '3' +services: + + tracetest: + image: kubeshop/tracetest:${TAG:-latest} + # uncommentig this line breaks the portability of this file, that is the base for the installer + # platform: linux/amd64 + volumes: + - type: bind + source: ./tracetest-config.yaml + target: /app/tracetest.yaml + - type: bind + source: ./tracetest-provision.yaml + target: /app/provision.yaml + command: --provisioning-file /app/provision.yaml + ports: + - 11633:11633 + extra_hosts: + - "host.docker.internal:host-gateway" + depends_on: + postgres: + condition: service_healthy + otel-collector: + condition: service_started + healthcheck: + test: ["CMD", "wget", "--spider", "localhost:11633"] + interval: 1s + timeout: 3s + retries: 60 + environment: + TRACETEST_DEV: ${TRACETEST_DEV} + + postgres: + image: postgres:14 + environment: + POSTGRES_PASSWORD: postgres + POSTGRES_USER: postgres + healthcheck: + test: pg_isready -U "$$POSTGRES_USER" -d "$$POSTGRES_DB" + interval: 1s + timeout: 5s + retries: 60 + + otel-collector: + image: otel/opentelemetry-collector:0.90.1 + command: + - "--config" + - "/otel-local-config.yaml" + volumes: + - ./collector.config.yaml:/otel-local-config.yaml + ports: + - 4317:4317 + environment: + - TRACETEST_ENDPOINT=tracetest:4317 diff --git a/examples/collector-filtering/tests/list-tests.yaml b/examples/collector-filtering/tests/list-tests.yaml new file mode 100644 index 0000000000..c29742a500 --- /dev/null +++ b/examples/collector-filtering/tests/list-tests.yaml @@ -0,0 +1,17 @@ +type: Test +spec: + id: e9c6cff9-974d-4263-8a23-22f1e9f975aa + name: List all tracetest tests + description: List all existing tests from tracetest API + trigger: + type: http + httpRequest: + url: http://localhost:11633/api/tests + method: GET + headers: + - key: Content-Type + value: application/json + specs: + - selector: span[tracetest.span.type="http" name="GET /api/tests"] + assertions: + - attr:tracetest.selected_spans.count = 1 diff --git a/examples/collector-filtering/tracetest-config.yaml b/examples/collector-filtering/tracetest-config.yaml new file mode 100644 index 0000000000..cbe3226feb --- /dev/null +++ b/examples/collector-filtering/tracetest-config.yaml @@ -0,0 +1,21 @@ +postgres: + host: postgres + user: postgres + password: postgres + port: 5432 + dbname: postgres + params: sslmode=disable + +telemetry: + exporters: + collector: + serviceName: tracetest + sampling: 100 # 100% + exporter: + type: collector + collector: + endpoint: otel-collector:4317 + +server: + telemetry: + exporter: collector diff --git a/examples/collector-filtering/tracetest-provision.yaml b/examples/collector-filtering/tracetest-provision.yaml new file mode 100644 index 0000000000..edf335a005 --- /dev/null +++ b/examples/collector-filtering/tracetest-provision.yaml @@ -0,0 +1,24 @@ +--- +type: PollingProfile +spec: + name: Default + strategy: periodic + default: true + periodic: + retryDelay: 5s + timeout: 10m + +--- +type: DataStore +spec: + name: OpenTelemetry Collector + type: otlp + default: true +--- +type: TestRunner +spec: + id: current + name: default + requiredGates: + - analyzer-score + - test-specs