feat(chart): configure OTLP export per signal

charts/substrate/README.md

@@ -43,3 +43,10 @@ See `values.yaml` for the full set; the important keys:
 | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` |
 | `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth |
 | `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces, metrics and the router access log |
+| `otel.traces.enabled` | `true` | Set to `false` to export no traces from the router; the Go components do not honor this yet |
+| `otel.traces.endpoint` | `""` | OTLP endpoint for traces, overriding `otel.endpoint` |
+| `otel.traces.samplingRatio` | `0.01` | Fraction of parentless requests that start a trace, applied to the Go components and the router |
+| `otel.metrics.enabled` | `true` | Sets the OTLP metrics exporter to `none`; the Go components do not honor this yet |
+| `otel.metrics.endpoint` | `""` | OTLP endpoint for metrics, overriding `otel.endpoint` |
+| `otel.logs.enabled` | `true` | Set to `false` to export no logs; the router access log is the only OTLP log source today |
+| `otel.logs.endpoint` | `""` | OTLP endpoint for logs, overriding `otel.endpoint` |

charts/substrate/templates/_helpers.tpl

@@ -78,6 +78,60 @@ Plaintext HTTP URL that clients use to reach atenet-router.
 {{- printf "http://%s.%s.svc:80" (include "substrate.fullname" (list "atenet-router" .)) .Release.Namespace -}}
 {{- end -}}
 
+{{/*
+OTLP endpoint a signal exports to, or empty when the signal is disabled or no
+endpoint resolves. The per-signal endpoint wins over the generic one, matching
+the precedence the OpenTelemetry SDK gives OTEL_EXPORTER_OTLP_<SIGNAL>_ENDPOINT
+over OTEL_EXPORTER_OTLP_ENDPOINT.
+
+Usage:
+  {{ include "substrate.otel.signalEndpoint" (list "traces" .) }}
+*/}}
+{{- define "substrate.otel.signalEndpoint" -}}
+{{- $signal := index . 0 -}}
+{{- $ctx := index . 1 -}}
+{{- $cfg := index $ctx.Values.otel $signal -}}
+{{- if $cfg.enabled -}}
+{{- $cfg.endpoint | default $ctx.Values.otel.endpoint -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+OTEL_* env entries for a Go component, as a list of "- name/value" items.
+Empty when nothing under .Values.otel is set, so callers can gate the env
+key on the result.
+
+Usage:
+  {{- with include "substrate.otel.env" . }}
+  {{- . | trim | nindent 8 }}
+  {{- end }}
+*/}}
+{{- define "substrate.otel.env" -}}
+{{- $otel := .Values.otel -}}
+{{- if $otel.endpoint }}
+- name: OTEL_EXPORTER_OTLP_ENDPOINT
+  value: {{ $otel.endpoint | quote }}
+{{- end }}
+{{- range $signal := list "traces" "metrics" "logs" }}
+{{- $cfg := index $otel $signal }}
+{{- if not $cfg.enabled }}
+{{- /* "none" is the SDK's own exporter name for "export nothing"; leaving the
+       endpoint unset would fall back to the SDK default of localhost:4317. */}}
+- name: OTEL_{{ upper $signal }}_EXPORTER
+  value: none
+{{- else if $cfg.endpoint }}
+- name: OTEL_EXPORTER_OTLP_{{ upper $signal }}_ENDPOINT
+  value: {{ $cfg.endpoint | quote }}
+{{- end }}
+{{- end }}
+{{- if include "substrate.otel.signalEndpoint" (list "traces" .) }}
+- name: OTEL_TRACES_SAMPLER
+  value: parentbased_traceidratio
+- name: OTEL_TRACES_SAMPLER_ARG
+  value: {{ $otel.traces.samplingRatio | quote }}
+{{- end }}
+{{- end -}}
+
 {{/*
 Build an image reference for a substrate component binary.
 

charts/substrate/templates/ate-api-server.yaml

@@ -108,9 +108,8 @@ spec:
               fieldPath: metadata.uid
         - name: OTEL_RESOURCE_ATTRIBUTES
           value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID)
-{{- if .Values.otel.endpoint }}
-        - name: OTEL_EXPORTER_OTLP_ENDPOINT
-          value: {{ .Values.otel.endpoint | quote }}
+{{- with include "substrate.otel.env" . }}
+{{- . | trim | nindent 8 }}
 {{- end }}
         - name: ATE_STORAGE_BACKEND
           value: {{ .Values.atelet.storageBackend | quote }}

charts/substrate/templates/ate-controller.yaml

@@ -79,10 +79,9 @@ spec:
         - "--ateapi-conn-spec=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443"
         - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem"
         - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem"
-{{- if .Values.otel.endpoint }}
+{{- with include "substrate.otel.env" . }}
         env:
-        - name: OTEL_EXPORTER_OTLP_ENDPOINT
-          value: {{ .Values.otel.endpoint | quote }}
+{{- . | trim | nindent 8 }}
 {{- end }}
         ports:
         - name: metrics

charts/substrate/templates/atelet.yaml

@@ -133,9 +133,8 @@ spec:
               fieldPath: metadata.uid
         - name: OTEL_RESOURCE_ATTRIBUTES
           value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),k8s.node.name=$(NODE_NAME),service.instance.id=$(POD_UID)
-{{- if .Values.otel.endpoint }}
-        - name: OTEL_EXPORTER_OTLP_ENDPOINT
-          value: {{ .Values.otel.endpoint | quote }}
+{{- with include "substrate.otel.env" . }}
+{{- . | trim | nindent 8 }}
 {{- end }}
         - name: ATE_STORAGE_BACKEND
           value: {{ .Values.atelet.storageBackend | quote }}

charts/substrate/templates/atenet-router.yaml

@@ -36,19 +36,25 @@ data:
       backend:
         poolMaxSize: 0
 
-{{- if .Values.otel.endpoint }}
+{{- $traces := include "substrate.otel.signalEndpoint" (list "traces" .) }}
+{{- $logs := include "substrate.otel.signalEndpoint" (list "logs" .) }}
+{{- if or $traces $logs }}
     frontendPolicies:
+{{- if $traces }}
       tracing:
-        host: $AGENTGATEWAY_OTLP_ADDRESS
+        host: $AGENTGATEWAY_OTLP_TRACES_ADDRESS
         protocol: grpc
-        randomSampling: 0.01
+        randomSampling: {{ .Values.otel.traces.samplingRatio }}
+{{- end }}
+{{- if $logs }}
       # Unsampled per-request access log. It carries the ate.* actor
-      # attribution, which the 1%-sampled traces cannot be aggregated on.
+      # attribution, which the sampled traces cannot be aggregated on.
       # `fields` stays unset so the OTLP sink inherits the full field set.
       accessLog:
         otlp:
-          host: $AGENTGATEWAY_OTLP_ADDRESS
+          host: $AGENTGATEWAY_OTLP_LOGS_ADDRESS
           protocol: grpc
+{{- end }}
 {{- end }}
 
     backends:
@@ -238,9 +244,8 @@ spec:
               fieldPath: metadata.uid
         - name: OTEL_RESOURCE_ATTRIBUTES
           value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID)
-{{- if .Values.otel.endpoint }}
-        - name: OTEL_EXPORTER_OTLP_ENDPOINT
-          value: {{ .Values.otel.endpoint | quote }}
+{{- with include "substrate.otel.env" . }}
+{{- . | trim | nindent 8 }}
 {{- end }}
         ports:
         - name: extproc
@@ -257,10 +262,19 @@ spec:
         args:
         - "-f"
         - "/etc/agentgateway/config.yaml"
-{{- if .Values.otel.endpoint }}
+{{- $traces := include "substrate.otel.signalEndpoint" (list "traces" .) }}
+{{- $logs := include "substrate.otel.signalEndpoint" (list "logs" .) }}
+{{- if or $traces $logs }}
+        # agentgateway takes host:port, not a URL, so the scheme is trimmed.
         env:
-        - name: AGENTGATEWAY_OTLP_ADDRESS
-          value: {{ trimPrefix "http://" .Values.otel.endpoint | quote }}
+{{- if $traces }}
+        - name: AGENTGATEWAY_OTLP_TRACES_ADDRESS
+          value: {{ trimPrefix "http://" $traces | quote }}
+{{- end }}
+{{- if $logs }}
+        - name: AGENTGATEWAY_OTLP_LOGS_ADDRESS
+          value: {{ trimPrefix "http://" $logs | quote }}
+{{- end }}
 {{- end }}
         ports:
         - name: http

charts/substrate/values.yaml

@@ -59,8 +59,26 @@ atelet:
 # Mounted via envFrom with optional=true. Created by the chart from these values.
 ateApiServerEnvVarsConfigMap: ate-api-server-envvars
 
+# OTLP export, following the OpenTelemetry SDK environment variable spec.
+# endpoint applies to every signal; a signal's own endpoint overrides it.
+# A signal exports only when it is enabled and resolves to an endpoint.
+#
+# traces.enabled and metrics.enabled set OTEL_<SIGNAL>_EXPORTER=none on the
+# Go components, which do not read it yet and keep exporting; the setting
+# takes effect on the router's agentgateway only. logs.enabled works in full,
+# since the agentgateway access log is the only OTLP log source.
 otel:
   endpoint: ""
+  traces:
+    enabled: true
+    endpoint: ""
+    samplingRatio: 0.01
+  metrics:
+    enabled: true
+    endpoint: ""
+  logs:
+    enabled: true
+    endpoint: ""
 
 image:
   registry: ghcr.io/kagent-dev/substrate