fix(otel): stop sending traces to api.openai.com and cleanup (#2400)

The Agents SDK registers a trace processor that POSTs everything to a
hardcoded `https://api.openai.com/v1/traces/ingest` using
`OPENAI_API_KEY`. It ignores `OPENAI_API_BASE`, so anyone behind other
proxies is shipping their gateway's key to OpenAI and getting 401s back.

`OpenAIAgentsInstrumentor` adds the OTel processor next to that one
rather than replacing it, and `set_tracing_disabled(True)` would silence
the OTel spans too, so we now drop the built-in processor before
instrumenting. Set `KAGENT_OPENAI_AGENTS_NATIVE_TRACING=true` to keep it
if you're using a real OpenAI key.

Using `OpenAIAgentsInstrumentor(replace_existing_processors=True)` as
suggested in review, with the pin bumped to `>=0.52.3,<0.53.0` since
that's when the kwarg landed. Doing it through the kwarg also means a
second `build()` can't wipe the OTel processor, since `instrument()`
no-ops once instrumented.

Cleaned up a couple of other smaller stuff:
- `Resource(...)` -> `Resource.create(...)`: the bare constructor
ignores `OTEL_RESOURCE_ATTRIBUTES` and drops `telemetry.sdk.*`, so
nobody could set `deployment.environment.name` or `service.version` at
all.
- Same bug in the Go ADK, `resource.New` starts empty so it needs
`WithFromEnv()` + `WithTelemetrySDK()`.
- `HTTPXClientInstrumentor().instrument(excluded_urls=...)` did nothing,
cleaned it up.
- `_a2a.py` read `self.config.kagent_url` but `KAgentConfig` only has
`url`, which blows up whenever `KAGENT_URL` isn't set.

Rebased onto main, which dropped the Gemini bullet from this PR since
main landed the same fix independently.

---------

Signed-off-by: krisztianfekete <git@krisztianfekete.org>

Author: krisztianfekete

go/adk/pkg/telemetry/tracing.go

@@ -71,17 +71,25 @@ func flushTimeout() time.Duration {
 	return 3 * time.Second
 }
 
+// newTelemetryResource builds the resource describing this service.
+func newTelemetryResource(ctx context.Context, serviceName string, serviceNamespace string) (*resource.Resource, error) {
+	return resource.New(ctx,
+		resource.WithFromEnv(),
+		resource.WithTelemetrySDK(),
+		resource.WithAttributes(
+			semconv.ServiceNameKey.String(serviceName),
+			semconv.ServiceNamespaceKey.String(serviceNamespace),
+		))
+}
+
 // Init initializes OpenTelemetry providers for Go ADK, sets global providers and
 // propagators, and returns a shutdown function.
 func Init(ctx context.Context, serviceName string, serviceNamespace string) (shutdown func(context.Context) error, enabled bool, err error) {
 	if !isTelemetryEnabled() {
 		return func(context.Context) error { return nil }, false, nil
 	}
 
-	telemetryResource, err := resource.New(ctx, resource.WithAttributes(
-		semconv.ServiceNameKey.String(serviceName),
-		semconv.ServiceNamespaceKey.String(serviceNamespace),
-	))
+	telemetryResource, err := newTelemetryResource(ctx, serviceName, serviceNamespace)
 	if err != nil {
 		return nil, true, err
 	}

go/adk/pkg/telemetry/tracing_test.go

@@ -65,3 +65,32 @@ func TestFlushTimeout(t *testing.T) {
 		})
 	}
 }
+
+// The resource must merge OTEL_RESOURCE_ATTRIBUTES and the telemetry.sdk.*
+// attributes. resource.New starts empty, so building it from WithAttributes
+// alone silently drops everything the environment supplies.
+func TestNewTelemetryResourceMergesEnvAttributes(t *testing.T) {
+	t.Setenv("OTEL_SERVICE_NAME", "should-not-win")
+	t.Setenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment.name=prod,service.version=1.4.2")
+
+	res, err := newTelemetryResource(context.Background(), "svc", "ns")
+	if err != nil {
+		t.Fatalf("newTelemetryResource: %v", err)
+	}
+
+	got := map[string]string{}
+	for _, kv := range res.Attributes() {
+		got[string(kv.Key)] = kv.Value.String()
+	}
+	for key, want := range map[string]string{
+		"deployment.environment.name": "prod",
+		"service.version":             "1.4.2",
+		"telemetry.sdk.language":      "go",
+		"service.name":                "svc",
+		"service.namespace":           "ns",
+	} {
+		if got[key] != want {
+			t.Errorf("attribute %s = %q, want %q (all: %v)", key, got[key], want, got)
+		}
+	}
+}

python/packages/kagent-core/src/kagent/core/tracing/_utils.py

@@ -236,7 +236,10 @@ def configure(
     tracing_enabled = os.getenv("OTEL_TRACING_ENABLED", "false").lower() == "true"
     logging_enabled = os.getenv("OTEL_LOGGING_ENABLED", "false").lower() == "true"
 
-    resource = Resource({"service.name": name, "service.namespace": namespace})
+    # Resource.create merges in OTEL_RESOURCE_ATTRIBUTES and the telemetry.sdk.*
+    # attributes; the bare constructor drops both, so deployment.environment.name,
+    # service.version and friends never reach the backend.
+    resource = Resource.create({"service.name": name, "service.namespace": namespace})
 
     # Configure tracing if enabled
     if tracing_enabled:
@@ -273,9 +276,11 @@ def configure(
 
         # Exclude agent-card endpoint from traces — this is used as a health
         # check endpoint (high-frequency polling requests) and has little
-        # diagnostic value.
+        # diagnostic value. Inbound only: HTTPXClientInstrumentor accepts no
+        # excluded_urls kwarg (newer releases read OTEL_PYTHON_HTTPX_EXCLUDED_URLS
+        # instead), so passing one here was silently dropped.
         _excluded_urls = ".*/\\.well-known/agent-card\\.json"
-        HTTPXClientInstrumentor().instrument(excluded_urls=_excluded_urls)
+        HTTPXClientInstrumentor().instrument()
         if fastapi_app:
             FastAPIInstrumentor().instrument_app(fastapi_app, excluded_urls=_excluded_urls)
             # Pre-response flushing is opt-in (the controller sets this on Agent

python/packages/kagent-core/tests/test_tracing_configure.py

@@ -83,6 +83,42 @@ def fake_instrument_google(logger_provider=None):
     assert instrument_calls["google_logger_provider"] is None
 
 
+def test_configure_resource_merges_otel_env_attributes(monkeypatch):
+    # OTEL_RESOURCE_ATTRIBUTES is the only way to set deployment.environment.name
+    # or service.version. The bare Resource() constructor ignores it entirely.
+    monkeypatch.setenv("OTEL_LOGGING_ENABLED", "false")
+    monkeypatch.setenv("OTEL_TRACING_ENABLED", "true")
+    monkeypatch.setenv("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment.name=prod,service.version=1.4.2")
+
+    captured = {}
+
+    class FakeTracerProvider:
+        def __init__(self, resource):
+            captured["resource"] = resource
+
+        def add_span_processor(self, processor):
+            pass
+
+    monkeypatch.setattr(_utils, "TracerProvider", FakeTracerProvider)
+    monkeypatch.setattr(_utils, "_create_span_exporter", lambda **kwargs: object())
+    monkeypatch.setattr(_utils, "BatchSpanProcessor", lambda exporter: object())
+    monkeypatch.setattr(_utils.trace, "set_tracer_provider", lambda provider: None)
+    monkeypatch.setattr(_utils, "HTTPXClientInstrumentor", lambda: SimpleNamespace(instrument=lambda **kw: None))
+    monkeypatch.setattr(_utils, "OpenAIInstrumentor", lambda **kwargs: SimpleNamespace(instrument=lambda **kw: None))
+    monkeypatch.setattr(_utils, "_instrument_anthropic", lambda *a, **kw: None)
+    monkeypatch.setattr(_utils, "_instrument_google_generativeai", lambda *a, **kw: None)
+
+    _utils.configure(name="test-agent", namespace="test-ns")
+
+    attributes = captured["resource"].attributes
+    # Identity stays under our control; env-supplied attributes come along.
+    assert attributes["service.name"] == "test-agent"
+    assert attributes["service.namespace"] == "test-ns"
+    assert attributes["deployment.environment.name"] == "prod"
+    assert attributes["service.version"] == "1.4.2"
+    assert attributes["telemetry.sdk.language"] == "python"
+
+
 def test_configure_all_disabled_skips_instrumentation(monkeypatch):
     monkeypatch.setenv("OTEL_LOGGING_ENABLED", "false")
     monkeypatch.setenv("OTEL_TRACING_ENABLED", "false")
@@ -279,7 +315,7 @@ def test_configure_gates_post_response_flush_on_env(monkeypatch, env_value, expe
     )
     monkeypatch.setattr(_utils, "OpenAIInstrumentor", lambda **kwargs: SimpleNamespace(instrument=lambda **kw: None))
     monkeypatch.setattr(_utils, "_instrument_anthropic", lambda *a, **kw: None)
-    monkeypatch.setattr(_utils, "_instrument_google_generativeai", lambda: None)
+    monkeypatch.setattr(_utils, "_instrument_google_generativeai", lambda *a, **kw: None)
 
     app = FastAPI()
     _utils.configure(name="test", namespace="test", fastapi_app=app)

python/packages/kagent-openai/pyproject.toml

@@ -15,7 +15,7 @@ dependencies = [
   "uvicorn>=0.20.0",
   "pydantic>=2.0.0",
   "typing-extensions>=4.16.0",
-  "opentelemetry-instrumentation-openai-agents>=0.50.0,<0.53.0"
+  "opentelemetry-instrumentation-openai-agents>=0.52.3,<0.53.0"
 ]
 
 [project.optional-dependencies]

python/packages/kagent-openai/src/kagent/openai/_a2a.py

@@ -80,6 +80,33 @@ def _configure_openai_client() -> None:
         logger.info(f"Configured OpenAI client with base URL: {openai_api_base}")
 
 
+def _configure_openai_agents_tracing() -> None:
+    """Export OpenAI Agents SDK traces through OpenTelemetry only.
+
+    The SDK's built-in processor POSTs to a hardcoded https://api.openai.com/v1/traces/ingest
+    using OPENAI_API_KEY, ignoring OPENAI_API_BASE, so agents behind an OpenAI-compatible
+    gateway (Azure AI Foundry, LiteLLM, ...) send that gateway's key to OpenAI and get 401s.
+    The instrumentor adds its processor alongside that one, and set_tracing_disabled(True)
+    would silence the OpenTelemetry spans too, so drop the built-in processor instead.
+
+    KAGENT_OPENAI_AGENTS_NATIVE_TRACING=true keeps it, for real OpenAI platform keys.
+    """
+    keep_native = os.getenv("KAGENT_OPENAI_AGENTS_NATIVE_TRACING", "false").strip().lower() == "true"
+    if keep_native:
+        logger.info("Keeping the OpenAI Agents SDK native trace exporter alongside OpenTelemetry")
+    else:
+        logger.info("Disabling the OpenAI Agents SDK native trace exporter; traces are exported via OpenTelemetry")
+
+    OpenAIAgentsInstrumentor(replace_existing_processors=not keep_native).instrument()
+
+    if os.getenv("OPENAI_AGENTS_DISABLE_TRACING", "false").strip().lower() in ("true", "1"):
+        logger.warning(
+            "OPENAI_AGENTS_DISABLE_TRACING is set, which switches off the Agents SDK tracing that the "
+            "OpenTelemetry instrumentation feeds on, so no agent spans will be exported. Unset it and rely "
+            "on KAGENT_OPENAI_AGENTS_NATIVE_TRACING=false (the default) to keep traces away from OpenAI."
+        )
+
+
 class KAgentApp:
     """FastAPI application builder for OpenAI Agents SDK with KAgent integration."""
 
@@ -121,7 +148,7 @@ def build(self) -> FastAPI:
 
         # Create HTTP client with KAgent backend
         http_client = httpx.AsyncClient(
-            base_url=kagent_url_override or self.config.kagent_url,
+            base_url=kagent_url_override or self.config.url,
         )
 
         # Create session factory
@@ -166,19 +193,20 @@ def build(self) -> FastAPI:
                 # OpenAIInstrumentor, whose SDK monkeypatch breaks Agents SDK streaming.
                 logger.info("Configuring tracing for KAgent OpenAI app")
                 configure_tracing(self.config.name, self.config.namespace, app, instrument_openai_client=False)
+                logger.info("Tracing configured for KAgent OpenAI app")
+            except Exception as e:
+                logger.error(f"Failed to configure tracing: {e}")
 
-                # Configure tracing for OpenAI Agents SDK
+            try:
                 tracing_enabled = os.getenv("OTEL_TRACING_ENABLED", "false").lower() == "true"
                 if tracing_enabled:
                     logger.info("Enabling OpenAI Agents SDK tracing")
-                    OpenAIAgentsInstrumentor().instrument()
+                    _configure_openai_agents_tracing()
                 else:
                     logger.info("Disabling OpenAI Agents SDK tracing")
                     set_tracing_disabled(True)
-
-                logger.info("Tracing configured for KAgent OpenAI app")
             except Exception as e:
-                logger.error(f"Failed to configure tracing: {e}")
+                logger.error(f"Failed to configure OpenAI Agents SDK tracing: {e}")
 
         # Add health check endpoints
         app.add_route("/health", methods=["GET"], route=health_check)

python/packages/kagent-openai/tests/test_tracing.py

@@ -0,0 +1,124 @@
+"""Tracing wiring for the OpenAI Agents SDK runtime."""
+
+import agents.tracing.setup as agents_tracing_setup
+import pytest
+from agents.tracing import get_trace_provider
+from agents.tracing.processors import BackendSpanExporter
+from agents.tracing.traces import NoOpTrace
+from opentelemetry.instrumentation.openai_agents import OpenAIAgentsInstrumentor
+from opentelemetry.instrumentation.openai_agents._hooks import OpenTelemetryTracingProcessor
+
+from kagent.openai._a2a import _configure_openai_agents_tracing
+
+
+def _reset():
+    instrumentor = OpenAIAgentsInstrumentor()
+    if instrumentor.is_instrumented_by_opentelemetry:
+        instrumentor.uninstrument()
+    # Cleared so the next get_trace_provider() rebuilds the SDK default provider,
+    # which is the state a fresh agent process starts in.
+    agents_tracing_setup.GLOBAL_TRACE_PROVIDER = None
+
+
+@pytest.fixture(autouse=True)
+def reset_agents_tracing(monkeypatch):
+    monkeypatch.delenv("KAGENT_OPENAI_AGENTS_NATIVE_TRACING", raising=False)
+    monkeypatch.delenv("OPENAI_AGENTS_DISABLE_TRACING", raising=False)
+    _reset()
+    yield
+    _reset()
+
+
+def _processors():
+    return list(get_trace_provider()._multi_processor._processors)
+
+
+def _exports_to_openai(processors):
+    return any(isinstance(getattr(p, "_exporter", None), BackendSpanExporter) for p in processors)
+
+
+def test_default_provider_exports_to_openai():
+    """Guards the premise: the SDK ships the api.openai.com exporter by default."""
+    assert _exports_to_openai(_processors())
+
+
+def test_native_exporter_dropped_by_default():
+    _configure_openai_agents_tracing()
+
+    processors = _processors()
+    assert not _exports_to_openai(processors)
+    assert [type(p) for p in processors] == [OpenTelemetryTracingProcessor]
+
+
+def test_repeat_configuration_keeps_opentelemetry_processor():
+    """BaseInstrumentor.instrument() no-ops once instrumented.
+
+    Anything that cleared processors outside _instrument would wipe the
+    OpenTelemetry processor on a second call with nothing to reinstall it.
+    """
+    _configure_openai_agents_tracing()
+    _configure_openai_agents_tracing()
+
+    processors = _processors()
+    assert not _exports_to_openai(processors)
+    assert [type(p) for p in processors] == [OpenTelemetryTracingProcessor]
+
+
+def test_spans_still_reach_opentelemetry():
+    """Dropping the native exporter must not disable SDK tracing altogether."""
+    _configure_openai_agents_tracing()
+
+    assert not isinstance(get_trace_provider().create_trace("test"), NoOpTrace)
+
+
+def test_native_exporter_kept_when_opted_in(monkeypatch):
+    monkeypatch.setenv("KAGENT_OPENAI_AGENTS_NATIVE_TRACING", "true")
+
+    _configure_openai_agents_tracing()
+
+    processors = _processors()
+    assert _exports_to_openai(processors)
+    assert any(isinstance(p, OpenTelemetryTracingProcessor) for p in processors)
+
+
+def test_build_drops_native_exporter_even_if_configure_tracing_fails(monkeypatch):
+    """A broken OTLP setup must not leave the SDK shipping traces to OpenAI."""
+    from agents import Agent
+    from kagent.core import KAgentConfig
+
+    from kagent.openai import _a2a
+
+    monkeypatch.setenv("OTEL_TRACING_ENABLED", "true")
+
+    def boom(*args, **kwargs):
+        raise RuntimeError("no collector")
+
+    monkeypatch.setattr(_a2a, "configure_tracing", boom)
+
+    agent_card = {
+        "name": "test",
+        "description": "test agent",
+        "version": "0.0.1",
+        "supportedInterfaces": [{"url": "http://localhost:8080", "protocolBinding": "JSONRPC"}],
+        "capabilities": {"streaming": True},
+        "defaultInputModes": ["text/plain"],
+        "defaultOutputModes": ["text/plain"],
+        "skills": [],
+    }
+    app = _a2a.KAgentApp(
+        agent=Agent(name="test"),
+        agent_card=agent_card,
+        config=KAgentConfig(url="http://localhost", name="test", namespace="test"),
+    )
+    app.build()
+
+    assert not _exports_to_openai(_processors())
+
+
+def test_warns_when_sdk_tracing_disabled_by_env(monkeypatch, caplog):
+    monkeypatch.setenv("OPENAI_AGENTS_DISABLE_TRACING", "1")
+
+    with caplog.at_level("WARNING"):
+        _configure_openai_agents_tracing()
+
+    assert "OPENAI_AGENTS_DISABLE_TRACING" in caplog.text