splunk · rcastley · Jul 9, 2026 · Jul 7, 2026 · Jul 7, 2026 · Jul 7, 2026
diff --git a/workshop/context-propagation/.env.example b/workshop/context-propagation/.env.example
@@ -0,0 +1,22 @@
+# Splunk Observability Cloud credentials
+# Copy this file to .env and replace placeholder values.
+
+# --- Splunk Observability Cloud ---
+SPLUNK_REALM=us0
+SPLUNK_ACCESS_TOKEN=YOUR_ORG_ACCESS_TOKEN
+
+# RUM browser agent (create in Splunk O11y: Data Management > RUM)
+SPLUNK_RUM_ACCESS_TOKEN=YOUR_RUM_ACCESS_TOKEN
+SPLUNK_RUM_APP_NAME=cosmic-observatory-shop
+SPLUNK_DEPLOYMENT_ENV=workshop-context-prop
+
+# --- Kubernetes / k3d ---
+# K3D_CLUSTER_NAME: local k3d cluster name (must match `k3d cluster list`)
+K3D_CLUSTER_NAME=cosmic-shop
+# CLUSTER_NAME: label shown in Splunk Infrastructure navigator (can differ from k3d name)
+CLUSTER_NAME=cosmic-shop-cluster
+REGISTRY=localhost:5111
+TAG=latest
+
+# --- Local docker-compose only ---
+VITE_DEV_PROXY=http://localhost:8080
diff --git a/workshop/context-propagation/.gitignore b/workshop/context-propagation/.gitignore
@@ -0,0 +1,6 @@
+node_modules/
+dist/
+.env
+deploy/k8s/otel.env
+.DS_Store
+*.log
diff --git a/workshop/context-propagation/Makefile b/workshop/context-propagation/Makefile
@@ -0,0 +1,59 @@
+.PHONY: setup-k3d build deploy collector up down logs fix-nginx fix-payment-gateway fix-rabbitmq import-images fix-images verify-infra-apm verify-apm cleanup cleanup-apps
+
+setup-k3d:
+	bash scripts/setup-k3d.sh
+
+build:
+	bash scripts/build-images.sh
+
+import-images:
+	bash scripts/import-images-k3d.sh
+
+fix-images:
+	bash scripts/fix-image-pull.sh
+
+deploy:
+	bash scripts/deploy.sh
+
+collector:
+	bash scripts/install-splunk-collector.sh
+
+up:
+	docker compose --env-file .env up --build -d
+
+down:
+	docker compose down
+
+logs:
+	docker compose logs -f order-api payment-gateway payment-api fulfillment-worker gateway
+
+fix-nginx:
+	kubectl apply -f deploy/k8s/gateway-config.yaml
+	kubectl -n cosmic-shop rollout restart deployment/gateway
+	kubectl -n cosmic-shop rollout status deployment/gateway --timeout=180s
+
+fix-payment-gateway:
+	bash scripts/build-images.sh payment-gateway
+	bash scripts/import-images-k3d.sh payment-gateway
+	kubectl -n cosmic-shop rollout restart deployment/payment-gateway
+	kubectl -n cosmic-shop rollout status deployment/payment-gateway --timeout=180s
+
+fix-rabbitmq:
+	bash scripts/build-images.sh payment-api fulfillment-worker
+	bash scripts/import-images-k3d.sh payment-api fulfillment-worker
+	kubectl -n cosmic-shop rollout restart deployment/payment-api
+	kubectl -n cosmic-shop rollout restart deployment/fulfillment-worker
+	kubectl -n cosmic-shop rollout status deployment/payment-api --timeout=180s
+	kubectl -n cosmic-shop rollout status deployment/fulfillment-worker --timeout=180s
+
+verify-infra-apm:
+	bash scripts/verify-infra-apm-config.sh
+
+verify-apm:
+	bash scripts/verify-apm-export.sh
+
+cleanup:
+	bash scripts/cleanup.sh
+
+cleanup-apps:
+	bash scripts/cleanup.sh --apps-only
diff --git a/workshop/context-propagation/README.md b/workshop/context-propagation/README.md
@@ -0,0 +1,47 @@
+# Context Propagation Workshop
+
+Distributed tracing breaks silently when proxies and message buses drop W3C trace context. This repository contains **Cosmic Observatory Shop** — an astronomy e-commerce demo with **three deliberate correlation breaks**:
+
+1. **Edge NGINX gateway** — strips headers from browser → order API
+2. **Payment gateway proxy** — instrumented Node.js proxy strips headers from order → payment API (visible in Splunk APM service map)
+3. **RabbitMQ** — async payment → fulfillment with no trace context in message headers
+
+## Quick start
+
+```bash
+cp .env.example .env
+# Edit .env with Splunk credentials
+
+make setup-k3d
+make build
+make deploy          # auto-installs Splunk OTel Collector if missing
+open http://localhost:30080
+```
+
+## Services
+
+| Service | Port | Stage |
+|---------|------|-------|
+| frontend | 30080 (k3d) | Shop UI + Splunk RUM |
+| gateway | — | Edge NGINX proxy |
+| order-api | 3001 | Order placement |
+| payment-gateway | 3004 | Payment proxy (APM-visible) |
+| payment-api | 3005 | Payment + RabbitMQ publish |
+| fulfillment-worker | 3006 | Async fulfillment consumer |
+| catalog-api | 3002 | Product catalog |
+| rabbitmq | 15672 | Management UI (guest/guest) |
+
+**RabbitMQ UI not loading?** Verify k3d mapped the port, then port-forward if needed:
+
+```bash
+docker ps --filter name=k3d-cosmic-shop-serverlb --format '{{.Ports}}'
+kubectl -n cosmic-shop port-forward svc/rabbitmq 15672:15672
+```
+
+## Cleanup
+
+```bash
+make cleanup              # Full teardown: k3d cluster, collector, images, Compose
+make cleanup-apps         # Remove apps only; keep k3d cluster
+```
+
diff --git a/workshop/context-propagation/deploy/helm/splunk-otel-values.yaml b/workshop/context-propagation/deploy/helm/splunk-otel-values.yaml
@@ -0,0 +1,15 @@
+# Splunk OTel Collector — base Helm values for k3d / generic Kubernetes.
+# Dynamic values (realm, token, clusterName, environment) are set via --set in
+# scripts/install-splunk-collector.sh from .env.
+#
+# For generic k3d clusters, leave distribution empty and set clusterName explicitly.
+
+splunkObservability:
+  metricsEnabled: true
+  tracesEnabled: true
+
+clusterReceiver:
+  enabled: true
+
+gateway:
+  enabled: false
diff --git a/workshop/context-propagation/deploy/k8s/catalog-api.yaml b/workshop/context-propagation/deploy/k8s/catalog-api.yaml
@@ -0,0 +1,86 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: catalog-api
+  namespace: cosmic-shop
+  labels:
+    app: catalog-api
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: catalog-api
+  template:
+    metadata:
+      labels:
+        app: catalog-api
+    spec:
+      containers:
+        - name: catalog-api
+          image: cosmic-shop/catalog-api:latest
+          imagePullPolicy: Never
+          ports:
+            - containerPort: 3002
+          envFrom:
+            - configMapRef:
+                name: otel-config
+          env:
+            - name: PORT
+              value: "3002"
+            - name: OTEL_SERVICE_NAME
+              value: catalog-api
+            - name: NODE_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.hostIP
+            - name: OTEL_EXPORTER_OTLP_ENDPOINT
+              value: http://$(NODE_IP):4318
+            - name: SPLUNK_ACCESS_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: splunk-otel
+                  key: accessToken
+            - name: SPLUNK_REALM
+              valueFrom:
+                secretKeyRef:
+                  name: splunk-otel
+                  key: realm
+            - name: K8S_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: K8S_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: OTEL_RESOURCE_ATTRIBUTES
+              value: "$(OTEL_RESOURCE_ATTRIBUTES),k8s.pod.name=$(K8S_POD_NAME),k8s.namespace.name=$(K8S_NAMESPACE)"
+          startupProbe:
+            httpGet:
+              path: /health
+              port: 3002
+            failureThreshold: 24
+            periodSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: 3002
+            periodSeconds: 5
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 3002
+            periodSeconds: 15
+            failureThreshold: 3
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: catalog-api
+  namespace: cosmic-shop
+spec:
+  selector:
+    app: catalog-api
+  ports:
+    - port: 3002
+      targetPort: 3002
diff --git a/workshop/context-propagation/deploy/k8s/frontend-api.yaml b/workshop/context-propagation/deploy/k8s/frontend-api.yaml
@@ -0,0 +1,92 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: frontend-api
+  namespace: cosmic-shop
+  labels:
+    app: frontend-api
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: frontend-api
+  template:
+    metadata:
+      labels:
+        app: frontend-api
+    spec:
+      containers:
+        - name: frontend-api
+          image: cosmic-shop/frontend-api:latest
+          imagePullPolicy: Never
+          ports:
+            - containerPort: 3007
+          envFrom:
+            - configMapRef:
+                name: otel-config
+          env:
+            - name: PORT
+              value: "3007"
+            - name: OTEL_SERVICE_NAME
+              value: frontend-api
+            - name: NODE_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.hostIP
+            - name: OTEL_EXPORTER_OTLP_ENDPOINT
+              value: http://$(NODE_IP):4318
+            - name: SPLUNK_ACCESS_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: splunk-otel
+                  key: accessToken
+            - name: SPLUNK_REALM
+              valueFrom:
+                secretKeyRef:
+                  name: splunk-otel
+                  key: realm
+            - name: K8S_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: K8S_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: OTEL_RESOURCE_ATTRIBUTES
+              value: "$(OTEL_RESOURCE_ATTRIBUTES),k8s.pod.name=$(K8S_POD_NAME),k8s.namespace.name=$(K8S_NAMESPACE)"
+            - name: CATALOG_API_URL
+              value: http://catalog-api:3002
+            - name: GATEWAY_URL
+              value: http://gateway:80
+            - name: PAYMENT_GATEWAY_URL
+              value: http://payment-gateway:3004
+          startupProbe:
+            httpGet:
+              path: /health
+              port: 3007
+            failureThreshold: 24
+            periodSeconds: 5
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: 3007
+            periodSeconds: 5
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 3007
+            periodSeconds: 15
+            failureThreshold: 3
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: frontend-api
+  namespace: cosmic-shop
+spec:
+  selector:
+    app: frontend-api
+  ports:
+    - port: 3007
+      targetPort: 3007
diff --git a/workshop/context-propagation/deploy/k8s/frontend.yaml b/workshop/context-propagation/deploy/k8s/frontend.yaml
@@ -0,0 +1,43 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: frontend
+  namespace: cosmic-shop
+  labels:
+    app: frontend
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: frontend
+  template:
+    metadata:
+      labels:
+        app: frontend
+    spec:
+      containers:
+        - name: frontend
+          image: cosmic-shop/frontend:latest
+          imagePullPolicy: Never
+          ports:
+            - containerPort: 80
+          readinessProbe:
+            httpGet:
+              path: /
+              port: 80
+            initialDelaySeconds: 3
+            periodSeconds: 5
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: frontend
+  namespace: cosmic-shop
+spec:
+  type: NodePort
+  selector:
+    app: frontend
+  ports:
+    - port: 80
+      targetPort: 80
+      nodePort: 30080