diff --git a/.env.example b/.env.example index a872972..7e9e6d9 100644 --- a/.env.example +++ b/.env.example @@ -6,5 +6,6 @@ GRAFANA_ADMIN_PASSWORD=change-me # GRAFANA_ROOT_URL=https://grafana.example.com GRAFANA_ROOT_URL=http://localhost:3000 +# Only needed for `just up-tunnel` (production exposure via Cloudflare Tunnel). # From: Cloudflare Zero Trust → Networks → Tunnels → → Configure → token CLOUDFLARE_TUNNEL_TOKEN= diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 7a8dfc9..a0ce903 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,7 +1,7 @@ version: 2 updates: - package-ecosystem: "docker-compose" - directory: "/" + directory: "/" schedule: interval: "weekly" groups: @@ -9,6 +9,11 @@ updates: patterns: - "*" + - package-ecosystem: "docker" + directory: "/demo" + schedule: + interval: "monthly" + - package-ecosystem: "github-actions" directory: "/" schedule: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..024130c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,26 @@ +# Thin wrapper around the `just` contract: everything CI runs, you can run +# locally with the same command. +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: extractions/setup-just@v4 + - run: cp .env.example .env + - run: just check + + smoke: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: extractions/setup-just@v4 + - run: cp .env.example .env + - run: just smoke + - run: just down diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..79bc055 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,42 @@ +# Changelog + +Notable changes to this stack. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); +versions follow [SemVer](https://semver.org/). + +## [0.1.0] - 2026-07-03 + +First tagged release: the stack is runnable, demoable, and validated in CI. + +### Added + +- `just demo`: one-command demo — an auto-instrumented FastAPI service under + constant load populates Grafana with correlated traces, metrics, and logs. +- `just check`: validation gate (compose syntax, Prometheus config + alert + rules, collector config, YAML, workflows, dashboard JSON), all in pinned + containers. Run by GitHub Actions on every push/PR, plus a `just smoke` + boot test. +- Provisioned **Service Health (RED)** dashboard driven by Tempo span + metrics — works for any service that sends traces. +- Prometheus alert rules: target down, OTel export failures, high span + error rate. +- README: architecture diagram, demo quickstart, screenshot, design + decisions. +- ADR 0001 recording the single-host OTLP-native architecture, and a + commented `compose.storage-s3.yml` stub documenting the storage + scale-out path. + +### Changed + +- `cloudflared` moved to an optional `compose.tunnel.yml` overlay and pinned + (was `:latest`); the core stack no longer needs a tunnel token to run. + +### Fixed + +- OTLP metrics ingestion: Prometheus 3.x renamed the OTLP receiver flag, + so the collector's metrics export 404ed and OTLP metrics were dropped. + +## [0.0.1] - 2026-06-30 + +Initial stack: OTel Collector → Loki / Tempo / Prometheus → Grafana wired +for cross-signal correlation, exposed via Cloudflare Tunnel, with RELab +dashboards. diff --git a/README.md b/README.md index 2805cab..810ea20 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,58 @@ # Observability stack -Central Grafana + Loki + Tempo + Prometheus + OTel Collector, wired together -with logs ↔ traces ↔ metrics correlation. Meant to run on one host and -receive telemetry from multiple projects over OTLP. +One host running Grafana, Loki, Tempo, Prometheus, and an OpenTelemetry +Collector, wired so logs, traces, and metrics cross-link. Point any number of +projects at it over OTLP and their telemetry lands in one place. + +```mermaid +flowchart LR + apps["Project apps
(OTLP gRPC 4317 / HTTP 4318)"] --> otel["OTel Collector
(ingestion gateway)"] + otel -- logs --> loki[Loki] + otel -- traces --> tempo[Tempo] + otel -- metrics --> prom[Prometheus] + tempo -- "span metrics (RED)" --> prom + loki --> grafana[Grafana] + tempo --> grafana + prom --> grafana + grafana --- cf["Cloudflare Tunnel
(optional overlay)"] +``` -## Layout +## Demo: see it work in one command +```sh +cp .env.example .env +just demo ``` -compose.yml # all services + +This starts the full stack plus a small auto-instrumented FastAPI service +under constant load (`compose.demo.yml`). Give it a minute, then open +Grafana at (admin / change-me): + +- **Dashboards → Service Health (RED)** — request rate, error rate, and + latency for the demo service. Latency dots are exemplars: click one to + open that exact trace in Tempo. +- **Explore → Loki** — `{service_name="demo-api"}`; expand an error line + and follow its `trace_id` to the trace. +- **Alerting → Alert rules** — stack-health and error-rate rules, + evaluated by Prometheus. + +![Service Health (RED) dashboard](docs/img/service-health.png) + +Tear it down with `just demo-down`. + +## Layout + +```sh +compose.yml # core services +compose.tunnel.yml # production overlay: Cloudflare Tunnel +compose.demo.yml # demo overlay: sample telemetry source +demo/ # the demo FastAPI service config/ otel-collector.yaml # ingestion gateway (OTLP in → Loki/Tempo/Prometheus out) loki.yaml # logs tempo.yaml # traces prometheus.yaml # metrics + alerts/ # Prometheus alert rules grafana/ # provisioned datasources + dashboard loader dashboards/ # drop JSON dashboards here; Grafana auto-loads them ``` @@ -21,11 +61,18 @@ dashboards/ # drop JSON dashboards here; Grafana auto-loads them ```sh cp .env.example .env # set GRAFANA_ADMIN_PASSWORD -just up +just up # core stack, local only +just up-tunnel # core stack + Cloudflare Tunnel (needs CLOUDFLARE_TUNNEL_TOKEN) ``` Grafana: (admin / whatever you set). +`just check` validates everything (compose files, Prometheus config and +alert rules, collector config, YAML, workflows, dashboard JSON) in pinned +containers — no host installs. CI runs the same command on every push and +PR, plus a smoke test that boots the stack and waits for Grafana to report +healthy. + ## Sending telemetry from a project ### Traces + metrics (OTLP) @@ -64,8 +111,35 @@ Logs carry labels for `service`, `env`, and `host` so you can filter in Grafana. Keep label cardinality low — don't add `user_id`, `request_id`, etc. as labels; use LogQL filters for those. +## Alerting + +Prometheus evaluates the rules in `config/alerts/` (target down, OTel +export failures, >5% span error rate) and Grafana surfaces them under +Alerting → Alert rules. There is deliberately no Alertmanager: on a +single-host stack, another service buys routing complexity before anyone +needs it. Add one (or a Grafana contact point) when alerts must page +someone. + ## Storage Everything persists to local Docker volumes (`loki_data`, `tempo_data`, `prometheus_data`, `grafana_data`). Swap Loki / Tempo storage to S3-compatible -(Backblaze B2, Cloudflare R2, Hetzner, MinIO) when you outgrow local disk. +(Backblaze B2, Cloudflare R2, Hetzner, MinIO) when you outgrow local disk — +`compose.storage-s3.yml` documents the concrete shape of that change, and +`docs/adr/0001-observability-stack.md` records the architecture rationale. + +## Design decisions + +Everything enters through one gateway — the collector — so a project +configures a single OTLP endpoint, and a backend can be swapped without +touching any app. It runs on one host on purpose: CML's telemetry is a handful +of services at single-digit requests per second, so distributed ingest would +add operational weight for no gain, while one compose file stays auditable by a +single person. When local disk runs short, S3-backed Loki and Tempo (above) +are the documented way out. + +Logs carry only low-cardinality labels — `service`, `env`, `host` — and +everything else is a query-time filter; high-cardinality labels are the usual +way a Loki install falls over. And because Tempo's metrics generator derives +RED metrics from spans, any service that sends traces gets the Service Health +dashboard whether or not it emits metrics of its own. diff --git a/compose.demo.yml b/compose.demo.yml new file mode 100644 index 0000000..0d1d317 --- /dev/null +++ b/compose.demo.yml @@ -0,0 +1,51 @@ +# Demo overlay: a tiny auto-instrumented FastAPI service plus a load +# generator, so the stack has real traces, metrics, and correlated logs +# to show. Not for production. +# +# just demo # core stack + this overlay +# just demo-down + +services: + demo-api: + build: ./demo + restart: unless-stopped + depends_on: [ otel-collector ] + environment: + OTEL_SERVICE_NAME: demo-api + OTEL_RESOURCE_ATTRIBUTES: env=demo + OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 + OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf + OTEL_TRACES_EXPORTER: otlp + OTEL_METRICS_EXPORTER: otlp + OTEL_LOGS_EXPORTER: otlp + # Ship Python log records via OTLP with trace context attached. + OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED: "true" + OTEL_METRIC_EXPORT_INTERVAL: "10000" + # Emit the *stable* HTTP semconv (http.server.request.duration in seconds, + # http.route) rather than the legacy names. Stable metric names are frozen, + # so the Service Health dashboard's app-SDK panel survives library bumps. + OTEL_SEMCONV_STABILITY_OPT_IN: http + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + + demo-load: + image: curlimages/curl:8.21.0 + restart: unless-stopped + depends_on: [ demo-api ] + command: + - sh + - -c + - | + while true; do + curl -s -o /dev/null demo-api:8000/ + curl -s -o /dev/null demo-api:8000/work + sleep 1 + done + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" diff --git a/compose.storage-s3.yml b/compose.storage-s3.yml new file mode 100644 index 0000000..921a8f7 --- /dev/null +++ b/compose.storage-s3.yml @@ -0,0 +1,54 @@ +# STUB — the storage scale-out path (not wired up; see ADR 0001). +# +# When local volumes stop fitting, Loki and Tempo move their object storage +# to any S3-compatible backend (Cloudflare R2, Backblaze B2, Hetzner, MinIO) +# without touching the collector, Prometheus, or any client project. +# +# This is deliberately a commented stub: it documents the concrete shape of +# the change so the README's scale-out claim is real, but it is not meant to +# be started until credentials and a bucket exist. To activate: +# +# 1. Create s3 variants of the configs, e.g. config/loki.s3.yaml replacing +# `common.storage.filesystem` with: +# +# common: +# storage: +# s3: +# endpoint: ${S3_ENDPOINT} # e.g. .r2.cloudflarestorage.com +# bucketnames: cml-loki +# access_key_id: ${S3_ACCESS_KEY_ID} +# secret_access_key: ${S3_SECRET_ACCESS_KEY} +# s3forcepathstyle: true +# +# and config/tempo.s3.yaml replacing `storage.trace.backend: local` with: +# +# storage: +# trace: +# backend: s3 +# s3: +# endpoint: ${S3_ENDPOINT} +# bucket: cml-tempo +# access_key: ${S3_ACCESS_KEY_ID} +# secret_key: ${S3_SECRET_ACCESS_KEY} +# +# 2. Uncomment the overlay below and run: +# docker compose -f compose.yml -f compose.storage-s3.yml up -d +# +# services: +# loki: +# volumes: +# - ./config/loki.s3.yaml:/etc/loki/loki.yaml:ro +# - loki_data:/loki +# environment: +# S3_ENDPOINT: ${S3_ENDPOINT:?set S3_ENDPOINT in .env} +# S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?set S3_ACCESS_KEY_ID in .env} +# S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?set S3_SECRET_ACCESS_KEY in .env} +# +# tempo: +# volumes: +# - ./config/tempo.s3.yaml:/etc/tempo/tempo.yaml:ro +# - tempo_data:/var/tempo +# environment: +# S3_ENDPOINT: ${S3_ENDPOINT:?set S3_ENDPOINT in .env} +# S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?set S3_ACCESS_KEY_ID in .env} +# S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?set S3_SECRET_ACCESS_KEY in .env} diff --git a/compose.tunnel.yml b/compose.tunnel.yml new file mode 100644 index 0000000..106896e --- /dev/null +++ b/compose.tunnel.yml @@ -0,0 +1,21 @@ +# Production overlay: expose Grafana (and the OTLP endpoints) via Cloudflare +# Tunnel. Requires CLOUDFLARE_TUNNEL_TOKEN in .env. +# +# just up-tunnel +# +# Token-based tunnel: routes are configured in the Cloudflare Zero Trust +# dashboard. Create a tunnel at dash.cloudflare.com → Zero Trust → Networks → +# Tunnels, then set the public hostnames there (e.g. grafana.example.com → +# http://grafana:3000, otlp.example.com → http://otel-collector:4318). + +services: + cloudflared: + image: cloudflare/cloudflared:2026.6.1 + restart: unless-stopped + depends_on: [ grafana, otel-collector ] + command: tunnel --no-autoupdate run --token ${CLOUDFLARE_TUNNEL_TOKEN:?set CLOUDFLARE_TUNNEL_TOKEN in .env} + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" diff --git a/compose.yml b/compose.yml index e1089a7..454c22f 100644 --- a/compose.yml +++ b/compose.yml @@ -55,13 +55,16 @@ services: restart: unless-stopped volumes: - ./config/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro + - ./config/alerts:/etc/prometheus/alerts:ro - prometheus_data:/prometheus command: - --config.file=/etc/prometheus/prometheus.yaml - --storage.tsdb.path=/prometheus - --storage.tsdb.retention.time=30d - --web.enable-remote-write-receiver - - --enable-feature=native-histograms,otlp-write-receiver,exemplar-storage + # Prometheus 3.x: OTLP ingestion is its own flag, no longer a feature flag. + - --web.enable-otlp-receiver + - --enable-feature=native-histograms,exemplar-storage logging: *default-logging grafana: @@ -82,18 +85,6 @@ services: GF_FEATURE_TOGGLES_ENABLE: traceqlEditor,metricsSummary,traceToProfiles logging: *default-logging - cloudflared: - image: cloudflare/cloudflared:latest - restart: unless-stopped - depends_on: [ grafana, otel-collector ] - # Token-based tunnel: routes are configured in the Cloudflare Zero Trust dashboard. - # Create a tunnel at dash.cloudflare.com → Zero Trust → Networks → Tunnels, - # then set the public hostnames there (e.g. grafana.example.com → http://grafana:3000, - # otlp.example.com → http://otel-collector:4318). - command: tunnel --no-autoupdate run --token ${CLOUDFLARE_TUNNEL_TOKEN:?set CLOUDFLARE_TUNNEL_TOKEN in .env} - logging: *default-logging - pull_policy: always - volumes: loki_data: tempo_data: diff --git a/config/alerts/stack.yaml b/config/alerts/stack.yaml new file mode 100644 index 0000000..c8016e3 --- /dev/null +++ b/config/alerts/stack.yaml @@ -0,0 +1,41 @@ +# Prometheus alert rules for the stack itself plus RED-style alerts on the +# span metrics Tempo derives from traces. Evaluated by Prometheus; visible in +# Grafana under Alerting → Alert rules (Prometheus datasource). +# No Alertmanager: single-host stack, alerts are surfaced in Grafana. +# Validated by `just check` (promtool check rules). + +groups: + - name: stack-health + rules: + - alert: TargetDown + expr: up == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Scrape target {{ $labels.job }} is down" + description: "Prometheus cannot scrape {{ $labels.instance }} (job {{ $labels.job }}) for 2 minutes." + + - alert: OtelExportFailures + expr: sum by (exporter) (rate(otelcol_exporter_send_failed_spans_total[5m])) > 0 + or sum by (exporter) (rate(otelcol_exporter_send_failed_metric_points_total[5m])) > 0 + or sum by (exporter) (rate(otelcol_exporter_send_failed_log_records_total[5m])) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "OTel Collector failing to export via {{ $labels.exporter }}" + description: "The collector has been failing to deliver telemetry to a backend for 5 minutes. Check `just logs otel-collector`." + + - name: service-red + rules: + - alert: HighErrorRate + expr: > + sum by (service) (rate(traces_spanmetrics_calls_total{status_code="STATUS_CODE_ERROR"}[5m])) + / sum by (service) (rate(traces_spanmetrics_calls_total[5m])) > 0.05 + for: 5m + labels: + severity: warning + annotations: + summary: "{{ $labels.service }} error rate above 5%" + description: "More than 5% of spans from {{ $labels.service }} report errors (from Tempo span metrics)." diff --git a/config/grafana/datasources.yaml b/config/grafana/datasources.yaml index 881abf3..5a2dcf4 100644 --- a/config/grafana/datasources.yaml +++ b/config/grafana/datasources.yaml @@ -25,6 +25,12 @@ datasources: matcherRegex: "[tT]race[_-]?[iI][dD][\"=:\\s]+([A-Fa-f0-9]+)" url: "$${__value.raw}" datasourceUid: tempo + # Same, for OTLP logs where trace_id is structured metadata, not body text. + - name: trace_id (otlp) + matcherType: label + matcherRegex: trace_id + url: "$${__value.raw}" + datasourceUid: tempo - name: Tempo type: tempo diff --git a/config/prometheus.yaml b/config/prometheus.yaml index fab2400..1d4c137 100644 --- a/config/prometheus.yaml +++ b/config/prometheus.yaml @@ -4,6 +4,9 @@ global: external_labels: origin: monitoring-host +rule_files: + - /etc/prometheus/alerts/*.yaml + # Apps push metrics via OTLP (through the collector). Scrape only infra we host. scrape_configs: - job_name: prometheus diff --git a/dashboards/service-health.json b/dashboards/service-health.json new file mode 100644 index 0000000..d0d6c73 --- /dev/null +++ b/dashboards/service-health.json @@ -0,0 +1,153 @@ +{ + "title": "Service Health (RED)", + "uid": "service-health", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { "from": "now-1h", "to": "now" }, + "graphTooltip": 1, + "tags": ["red", "otlp"], + "templating": { + "list": [ + { + "name": "service", + "label": "Service", + "type": "query", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "definition": "label_values(traces_spanmetrics_calls_total, service)", + "query": { "qryType": 1, "query": "label_values(traces_spanmetrics_calls_total, service)", "refId": "var" }, + "refresh": 2, + "sort": 1, + "current": { "text": "demo-api", "value": "demo-api" }, + "options": [], + "hide": 0 + } + ] + }, + "panels": [ + { + "type": "timeseries", + "title": "Request Rate by Endpoint", + "description": "Server spans per second, from Tempo span metrics — works for any service that sends traces.", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "id": 1, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "sum by (span_name) (rate(traces_spanmetrics_calls_total{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval]))", + "legendFormat": "{{span_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { "lineWidth": 2, "fillOpacity": 10 } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Error Rate", + "description": "Share of server spans with STATUS_CODE_ERROR.", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "id": 2, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "100 * sum(rate(traces_spanmetrics_calls_total{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\", status_code=\"STATUS_CODE_ERROR\"}[$__rate_interval])) / sum(rate(traces_spanmetrics_calls_total{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval]))", + "legendFormat": "errors", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, + "color": { "mode": "fixed", "fixedColor": "red" }, + "custom": { "lineWidth": 2, "fillOpacity": 10 }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }] } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "Latency (from traces, with exemplars)", + "description": "Quantiles over Tempo span-metrics latency. Dots are exemplars — click one to open the exact trace in Tempo.", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "id": 3, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))", + "legendFormat": "p50", + "refId": "A", + "exemplar": true + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))", + "legendFormat": "p95", + "refId": "B", + "exemplar": true + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (rate(traces_spanmetrics_latency_bucket{service=\"$service\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))", + "legendFormat": "p99", + "refId": "C", + "exemplar": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { "lineWidth": 2, "fillOpacity": 5 } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "title": "HTTP Server Duration p95 (app SDK metrics)", + "description": "From the service's own OTLP metrics (OTel SDK), i.e. the collector → Prometheus OTLP path.", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "id": 4, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{job=\"$service\"}[$__rate_interval])))", + "legendFormat": "{{http_route}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { "lineWidth": 2, "fillOpacity": 10 } + }, + "overrides": [] + } + }, + { + "type": "logs", + "title": "Logs", + "description": "OTLP logs for the service. Expand a line and follow trace_id to jump to the trace in Tempo.", + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 16 }, + "id": 5, + "datasource": { "type": "loki", "uid": "loki" }, + "targets": [ + { + "expr": "{service_name=\"$service\"}", + "refId": "A" + } + ], + "options": { + "showTime": true, + "wrapLogMessage": true, + "enableLogDetails": true, + "sortOrder": "Descending" + } + } + ] +} diff --git a/demo/Dockerfile b/demo/Dockerfile new file mode 100644 index 0000000..d3c9bee --- /dev/null +++ b/demo/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.13-slim + +RUN pip install --no-cache-dir \ + fastapi==0.139.0 \ + uvicorn==0.49.0 \ + opentelemetry-distro==0.64b0 \ + opentelemetry-exporter-otlp==1.43.0 \ + opentelemetry-instrumentation-fastapi==0.64b0 + +WORKDIR /app +COPY app.py . + +CMD ["opentelemetry-instrument", "uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/demo/__init__.py b/demo/__init__.py new file mode 100644 index 0000000..fd3aec1 --- /dev/null +++ b/demo/__init__.py @@ -0,0 +1 @@ +"""Demo telemetry source for the monitoring stack (see compose.demo.yml).""" diff --git a/demo/app.py b/demo/app.py new file mode 100644 index 0000000..a43c6ac --- /dev/null +++ b/demo/app.py @@ -0,0 +1,34 @@ +"""Minimal FastAPI service for the demo overlay. + +All telemetry (traces, metrics, logs with trace context) comes from OTel +auto-instrumentation — see compose.demo.yml. No OTel code needed here. +""" + +import logging +import random +import time + +from fastapi import FastAPI, HTTPException + +ERROR_RATE = 0.1 # fixed error rate, enough to light up RED panels + +logging.basicConfig(level=logging.INFO) # root logger defaults to WARNING; we want the INFO lines too +log = logging.getLogger("demo-api") +app = FastAPI() + + +@app.get("/") +def root() -> dict[str, bool]: + """Fast, always-successful endpoint.""" + return {"ok": True} + + +@app.get("/work") +def work() -> dict[str, bool]: + """Simulate variable-latency work that sometimes fails.""" + time.sleep(random.uniform(0.02, 0.3)) # noqa: S311 — not crypto, just jitter + if random.random() < ERROR_RATE: # noqa: S311 + log.error("work failed: upstream flaked") + raise HTTPException(status_code=500, detail="upstream flaked") + log.info("work done") + return {"ok": True} diff --git a/docs/adr/0001-observability-stack.md b/docs/adr/0001-observability-stack.md new file mode 100644 index 0000000..b69f096 --- /dev/null +++ b/docs/adr/0001-observability-stack.md @@ -0,0 +1,48 @@ +# ADR 0001: Single-host OTLP-native observability stack + +Date: 2026-07-03. Status: accepted (records a decision already in production). + +## Context + +CML runs several long-lived research platforms (RELab and others) that need +their logs, traces, and metrics in one place. The volume is modest — a handful +of services at single-digit requests per second — but one small team maintains +all of it, so whatever we run must stay cheap and auditable. + +## Decision + +Run one Docker Compose stack on a single host: an OpenTelemetry Collector as +the only ingestion endpoint, fanning out to Loki (logs), Tempo (traces), and +Prometheus (metrics), with Grafana on top. Expose Grafana and the OTLP +endpoints via Cloudflare Tunnel; bind everything else to `127.0.0.1`. + +- **OTLP-native:** one endpoint, one protocol. Swap a backend or move it to a + hosted equivalent without touching any application. +- **Single-host:** one compose file one person can audit and rebuild. + Distributed ingest (Kubernetes, Mimir, multi-tenant Loki) would add operational + overhead for no benefit at this volume. +- **Traces as the primary signal:** Tempo's metrics generator derives RED + metrics and a service graph from spans, so a service that only sends traces + still gets dashboards and error-rate alerting. + +## Alternatives considered + +- **Grafana Cloud / hosted SaaS:** less to operate, but a recurring bill, + data-residency questions for research data, and little institutional + infrastructure knowledge to show for it. +- **Per-project stacks:** no shared endpoint to maintain, but each project + duplicates its own storage and Grafana, and cross-project correlation + becomes impossible. +- **ELK / OpenSearch:** heavier to run and log-centric, with a weaker native + OTLP and trace-correlation story than the Grafana stack. + +## Consequences + +- The host is a single point of failure — acceptable, because the monitored + platforms degrade gracefully when telemetry stops (OTLP export is + fire-and-forget) and the stack rebuilds from this repo in minutes. +- Local disk bounds retention (30d logs/metrics, 7d traces). The escape hatch, + reached before any move to distributed ingest, is S3-compatible storage for + Loki and Tempo — see `compose.storage-s3.yml`. +- Every image is pinned and validated by `just check` in CI, so the stack + stays reproducible. diff --git a/docs/img/service-health.png b/docs/img/service-health.png new file mode 100644 index 0000000..a79f364 Binary files /dev/null and b/docs/img/service-health.png differ diff --git a/justfile b/justfile index f80ccdb..61116ac 100644 --- a/justfile +++ b/justfile @@ -3,11 +3,24 @@ set dotenv-load default: @just --list +# Core stack (no tunnel; Grafana at http://localhost:3000) up: docker compose up -d +# Core stack + Cloudflare Tunnel (production; needs CLOUDFLARE_TUNNEL_TOKEN) +up-tunnel: + docker compose -f compose.yml -f compose.tunnel.yml up -d + down: - docker compose down + docker compose down --remove-orphans + +# Core stack + a demo telemetry source (see compose.demo.yml), then look at +# Grafana: http://localhost:3000 +demo: + docker compose -f compose.yml -f compose.demo.yml up -d --build + +demo-down: + docker compose -f compose.yml -f compose.demo.yml down logs service="": docker compose logs -f {{service}} @@ -24,3 +37,24 @@ pull: # Tail a service's logs as JSON, decoded. Useful before Grafana is set up. tail service: docker compose logs -f --no-log-prefix {{service}} | jq -R 'fromjson? // .' + +# Validate everything. All validators run in containers — no host installs. +check: + docker compose config -q + CLOUDFLARE_TUNNEL_TOKEN=dummy docker compose -f compose.yml -f compose.tunnel.yml config -q + docker compose -f compose.yml -f compose.demo.yml config -q + docker run --rm -v ./config/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro -v ./config/alerts:/etc/prometheus/alerts:ro --entrypoint promtool prom/prometheus:v3.11.2 check config /etc/prometheus/prometheus.yaml + docker run --rm -v ./config/otel-collector.yaml:/etc/otelcol/config.yaml:ro otel/opentelemetry-collector-contrib:0.150.1 validate --config=/etc/otelcol/config.yaml + docker run --rm -v .:/code:ro pipelinecomponents/yamllint:0.35.13 yamllint -d relaxed . + docker run --rm -v .:/repo:ro -w /repo rhysd/actionlint:1.7.12 -color + jq empty dashboards/*.json + +# Format YAML in place (needs yamlfmt on the host; optional). +fmt: + yamlfmt . + +# Boot the core stack and wait until Grafana reports healthy. Used by CI. +smoke: + docker compose up -d + n=0; until curl -sf http://localhost:3000/api/health >/dev/null; do n=$((n+3)); [ $n -ge 120 ] && { echo "Grafana not healthy after 120s" >&2; exit 1; }; sleep 3; done + @echo "Grafana healthy"