chore: adding observability stack example (#3578)

* chore: adding observability stack example

* adding README

* Apply suggestions from code review

Co-authored-by: Julianne Fermi <julianne@kubeshop.io>

* update example

* Apply suggestions from code review

Co-authored-by: Adnan Rahić <adnan@kubeshop.io>

* update readme

* add suggestion

---------

Co-authored-by: Julianne Fermi <julianne@kubeshop.io>
Co-authored-by: Adnan Rahić <adnan@kubeshop.io>

examples/observability-stack/.gitignore

@@ -0,0 +1 @@
+node_modules

examples/observability-stack/README.md

@@ -0,0 +1,31 @@
+# Observability Stack Example
+
+This folder has minimal code to run a local observability stack, plug an API into it, and do some trace-based tests.
+
+To run the observability stack and the local API, execute:
+
+```sh
+# run the observability stack 
+docker compose up -d
+
+# install dependencies and run API
+npm install
+npm run with-telemetry
+```
+
+To run it with trace-based tests, execute:
+
+```sh
+# run the observability stack 
+export TRACETEST_API_KEY="{{Your Tracetest.io Token}}"
+docker compose -f ./docker-compose.yaml -f docker-compose.tracetest.yaml up -d
+
+# install dependencies and run API
+npm install
+npm run with-telemetry
+```
+
+And then run a test with:
+```sh
+tracetest run test -f ./test-api.yaml
+```

examples/observability-stack/app.instrumentation.js

@@ -0,0 +1,21 @@
+const opentelemetry = require('@opentelemetry/sdk-node')
+const { getNodeAutoInstrumentations } = require('@opentelemetry/auto-instrumentations-node')
+const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-grpc')
+const { OTLPMetricExporter } = require('@opentelemetry/exporter-metrics-otlp-grpc')
+const { PeriodicExportingMetricReader } = require('@opentelemetry/sdk-metrics')
+const grpc = require('@grpc/grpc-js')
+
+const exporterConfig = {
+  url: 'localhost:4317',
+  credentials: grpc.ChannelCredentials.createInsecure()
+}
+
+const sdk = new opentelemetry.NodeSDK({
+  metricReader: new PeriodicExportingMetricReader({
+    exporter: new OTLPMetricExporter(exporterConfig)
+  }),
+  traceExporter: new OTLPTraceExporter(exporterConfig),
+  instrumentations: [getNodeAutoInstrumentations()],
+  serviceName: 'test-api',
+})
+sdk.start()

examples/observability-stack/app.js

@@ -0,0 +1,12 @@
+const express = require("express")
+const app = express()
+
+app.get("/", (req, res) => {
+  setTimeout(() => {
+    res.send("Hello World")
+  }, 1000);
+})
+
+app.listen(8080, () => {
+  console.log(`Listening for requests on http://localhost:8080`)
+})

examples/observability-stack/config/grafana.datasource.yaml

@@ -0,0 +1,32 @@
+# config file version
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    uid: prometheus
+    access: proxy
+    orgId: 1
+    url: http://prometheus:9090
+    basicAuth: false
+    isDefault: false
+    version: 1
+    editable: false
+    jsonData:
+      httpMethod: GET
+
+  - name: Tempo
+    type: tempo
+    access: proxy
+    orgId: 1
+    url: http://tempo:3200
+    basicAuth: false
+    isDefault: true
+    version: 1
+    editable: false
+    apiVersion: 1
+    uid: tempo
+    jsonData:
+      httpMethod: GET
+      serviceMap:
+        datasourceUid: prometheus

examples/observability-stack/config/otel-collector.config.yaml

@@ -0,0 +1,40 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+      http:
+        cors:
+          allowed_origins:
+            - "http://*"
+            - "https://*"
+
+processors:
+  batch:
+    timeout: 100ms
+
+exporters:
+  debug:
+    verbosity: detailed
+
+  otlp/tempo:
+    endpoint: tempo:4317
+    tls:
+      insecure: true
+
+  prometheus:
+    endpoint: 0.0.0.0:8889
+
+extensions:
+  health_check: {}
+
+service:
+  pipelines:
+    metrics:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [debug, prometheus]
+
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [debug, otlp/tempo]

examples/observability-stack/config/prometheus.config.yaml

@@ -0,0 +1,13 @@
+global:
+  scrape_interval:     15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: otel-collector
+    static_configs:
+      - targets: ['otel-collector:8889']
+      - targets: ['otel-collector:8888']
+
+tracing:
+  endpoint: otel-collector:4317
+  insecure: true

examples/observability-stack/config/tempo.config.yaml

@@ -0,0 +1,52 @@
+stream_over_http_enabled: true
+
+server:
+  http_listen_port: 3200
+  log_level: info
+
+query_frontend:
+  search:
+    duration_slo: 5s
+    throughput_bytes_slo: 1.073741824e+09
+  trace_by_id:
+    duration_slo: 5s
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        http:
+          endpoint: 0.0.0.0:4318
+        grpc:
+          endpoint: 0.0.0.0:4317
+
+ingester:
+  max_block_duration: 5m               # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
+
+compactor:
+  compaction:
+    block_retention: 1h                # overall Tempo trace retention. set for demo purposes
+
+metrics_generator:
+  registry:
+    external_labels:
+      source: tempo
+      cluster: docker-compose
+  storage:
+    path: /tmp/tempo/generator/wal
+    remote_write:
+      - url: http://prometheus:9090/api/v1/write
+        send_exemplars: true
+
+storage:
+  trace:
+    backend: local                     # backend configuration to use
+    wal:
+      path: /tmp/tempo/wal             # where to store the the wal locally
+    local:
+      path: /tmp/tempo/blocks
+
+overrides:
+  defaults:
+    metrics_generator:
+      processors: [service-graphs, span-metrics] # enables metrics generator

examples/observability-stack/docker-compose.tracetest.yaml

@@ -0,0 +1,13 @@
+version: "3.7"
+
+services:
+  tracetest:
+    image: kubeshop/tracetest-agent:latest
+    platform: linux/amd64
+    depends_on:
+      otel-collector:
+        condition: service_started
+    environment:
+      TRACETEST_API_KEY: ${TRACETEST_API_KEY}
+    extra_hosts:
+      - "host.docker.internal:host-gateway"

examples/observability-stack/docker-compose.yaml

@@ -0,0 +1,50 @@
+version: "3.7"
+
+services:
+  grafana:
+    image: grafana/grafana:10.2.3
+    user: "472"
+    depends_on:
+      - prometheus
+      - tempo
+      - otel-collector
+    ports:
+      - 33000:33000
+    environment:
+      - GF_SERVER_HTTP_PORT=33000
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+      - GF_AUTH_DISABLE_LOGIN_FORM=true
+    volumes:
+      - type: bind
+        source: ./config/grafana.datasource.yaml
+        target: /etc/grafana/provisioning/datasources/datasources.yaml
+
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:0.92.0
+    command:
+      - "--config"
+      - "/otel-local-config.yaml"
+    volumes:
+      - ./config/otel-collector.config.yaml:/otel-local-config.yaml
+    ports:
+      - 4317:4317
+
+  tempo:
+    image: grafana/tempo:2.3.1
+    command: ["-config.file=/etc/tempo.yaml"]
+    volumes:
+      - type: bind
+        source: ./config/tempo.config.yaml
+        target: /etc/tempo.yaml
+
+  prometheus:
+    image: prom/prometheus:v2.49.1
+    command:
+      - --config.file=/etc/prometheus.yaml
+      - --web.enable-remote-write-receiver
+      - --enable-feature=exemplar-storage
+    volumes:
+      - type: bind
+        source: ./config/prometheus.config.yaml
+        target: /etc/prometheus.yaml