From 7f42d79b56e2df00b2ba916123b08ba7d238d20b Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Thu, 28 May 2026 10:57:35 +0100 Subject: [PATCH 1/2] fix(core): external trace context leaks across warm-started runs --- .../warm-start-external-trace-context-leak.md | 5 + packages/core/src/v3/otel/tracingSDK.ts | 93 +++++++++---------- .../test/externalSpanExporterWrapper.test.ts | 82 ++++++++++++++++ 3 files changed, 131 insertions(+), 49 deletions(-) create mode 100644 .changeset/warm-start-external-trace-context-leak.md create mode 100644 packages/core/test/externalSpanExporterWrapper.test.ts diff --git a/.changeset/warm-start-external-trace-context-leak.md b/.changeset/warm-start-external-trace-context-leak.md new file mode 100644 index 00000000000..84f91de7689 --- /dev/null +++ b/.changeset/warm-start-external-trace-context-leak.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/core": patch +--- + +Fix external trace context leaking across runs on warm-started workers with `processKeepAlive` enabled. Every subsequent run's attempt span was being exported with the first run's `traceId` and `parentSpanId`, breaking causal-chain navigation in external APM tools. Runs without an external trace context are unaffected. diff --git a/packages/core/src/v3/otel/tracingSDK.ts b/packages/core/src/v3/otel/tracingSDK.ts index a301946c755..f7814e8cca1 100644 --- a/packages/core/src/v3/otel/tracingSDK.ts +++ b/packages/core/src/v3/otel/tracingSDK.ts @@ -171,13 +171,12 @@ export class TracingSDK { ); const externalTraceId = idGenerator.generateTraceId(); - const externalTraceContext = traceContext.getExternalTraceContext(); for (const exporter of config.exporters ?? []) { spanProcessors.push( getEnvVar("TRIGGER_OTEL_BATCH_PROCESSING_ENABLED") === "1" ? new BatchSpanProcessor( - new ExternalSpanExporterWrapper(exporter, externalTraceId, externalTraceContext), + new ExternalSpanExporterWrapper(exporter, externalTraceId), { maxExportBatchSize: parseInt( getEnvVar("TRIGGER_OTEL_SPAN_MAX_EXPORT_BATCH_SIZE") ?? "64" @@ -192,7 +191,7 @@ export class TracingSDK { } ) : new SimpleSpanProcessor( - new ExternalSpanExporterWrapper(exporter, externalTraceId, externalTraceContext) + new ExternalSpanExporterWrapper(exporter, externalTraceId) ) ); } @@ -245,11 +244,7 @@ export class TracingSDK { logProcessors.push( getEnvVar("TRIGGER_OTEL_BATCH_PROCESSING_ENABLED") === "1" ? new BatchLogRecordProcessor( - new ExternalLogRecordExporterWrapper( - externalLogExporter, - externalTraceId, - externalTraceContext - ), + new ExternalLogRecordExporterWrapper(externalLogExporter, externalTraceId), { maxExportBatchSize: parseInt( getEnvVar("TRIGGER_OTEL_LOG_MAX_EXPORT_BATCH_SIZE") ?? "64" @@ -264,11 +259,7 @@ export class TracingSDK { } ) : new SimpleLogRecordProcessor( - new ExternalLogRecordExporterWrapper( - externalLogExporter, - externalTraceId, - externalTraceContext - ) + new ExternalLogRecordExporterWrapper(externalLogExporter, externalTraceId) ) ); } @@ -417,23 +408,23 @@ function setLogLevel(level: TracingDiagnosticLogLevel) { diag.setLogger(new DiagConsoleLogger(), diagLogLevel); } -class ExternalSpanExporterWrapper { - private readonly _isExternallySampled: boolean; - +export class ExternalSpanExporterWrapper { constructor( private underlyingExporter: SpanExporter, - private externalTraceId: string, - private externalTraceContext: - | { traceId: string; spanId: string; traceFlags: number; tracestate?: string } - | undefined - ) { - this._isExternallySampled = externalTraceContext - ? isTraceFlagSampled(externalTraceContext.traceFlags) - : !!externalTraceId; - } + private externalTraceId: string + ) {} private transformSpan(span: ReadableSpan): ReadableSpan | undefined { - if (!this._isExternallySampled) { + // Read external context live, so per-run reassignment of + // standardTraceContextManager.traceContext is honoured on warm-started + // workers that reuse a single TracingSDK across runs. + const externalTraceContext = traceContext.getExternalTraceContext(); + + const isExternallySampled = externalTraceContext + ? isTraceFlagSampled(externalTraceContext.traceFlags) + : !!this.externalTraceId; + + if (!isExternallySampled) { return; } @@ -441,8 +432,8 @@ class ExternalSpanExporterWrapper { return; } - const externalTraceId = this.externalTraceContext - ? this.externalTraceContext.traceId + const externalTraceId = externalTraceContext + ? externalTraceContext.traceId : this.externalTraceId; const isAttemptSpan = span.attributes[SemanticInternalAttributes.SPAN_ATTEMPT]; @@ -457,15 +448,15 @@ class ExternalSpanExporterWrapper { }; } - if (isAttemptSpan && this.externalTraceContext) { + if (isAttemptSpan && externalTraceContext) { parentSpanContext = { ...parentSpanContext, traceId: externalTraceId, - spanId: this.externalTraceContext.spanId, - traceState: this.externalTraceContext.tracestate - ? new TraceState(this.externalTraceContext.tracestate) + spanId: externalTraceContext.spanId, + traceState: externalTraceContext.tracestate + ? new TraceState(externalTraceContext.tracestate) : undefined, - traceFlags: this.externalTraceContext.traceFlags, + traceFlags: externalTraceContext.traceFlags, }; } else if (isAttemptSpan) { parentSpanContext = undefined; @@ -502,28 +493,27 @@ class ExternalSpanExporterWrapper { } class ExternalLogRecordExporterWrapper { - private readonly _isExternallySampled: boolean; - constructor( private underlyingExporter: LogRecordExporter, - private externalTraceId: string, - private externalTraceContext: - | { traceId: string; spanId: string; tracestate?: string; traceFlags: number } - | undefined - ) { - this._isExternallySampled = externalTraceContext - ? isTraceFlagSampled(externalTraceContext.traceFlags) - : !!externalTraceId; - } + private externalTraceId: string + ) {} export(logs: any[], resultCallback: (result: any) => void): void { - if (!this._isExternallySampled) { + const externalTraceContext = traceContext.getExternalTraceContext(); + + const isExternallySampled = externalTraceContext + ? isTraceFlagSampled(externalTraceContext.traceFlags) + : !!this.externalTraceId; + + if (!isExternallySampled) { this.underlyingExporter.export([], resultCallback); return; } - const modifiedLogs = logs.map(this.transformLogRecord.bind(this)); + const modifiedLogs = logs.map((log) => + this.transformLogRecord(log, externalTraceContext) + ); this.underlyingExporter.export(modifiedLogs, resultCallback); } @@ -532,11 +522,16 @@ class ExternalLogRecordExporterWrapper { return this.underlyingExporter.shutdown(); } - transformLogRecord(logRecord: ReadableLogRecord): ReadableLogRecord { + transformLogRecord( + logRecord: ReadableLogRecord, + externalTraceContext: + | { traceId: string; spanId: string; tracestate?: string; traceFlags: number } + | undefined + ): ReadableLogRecord { // Capture externalTraceId for use within the proxy's scope. // Use externalTraceContext.traceId if available, otherwise fall back to generated externalTraceId - const externalTraceId = this.externalTraceContext - ? this.externalTraceContext.traceId + const externalTraceId = externalTraceContext + ? externalTraceContext.traceId : this.externalTraceId; // If there's no spanContext, or if the externalTraceId is not set, return the original logRecord. diff --git a/packages/core/test/externalSpanExporterWrapper.test.ts b/packages/core/test/externalSpanExporterWrapper.test.ts new file mode 100644 index 00000000000..3e2d3f2329a --- /dev/null +++ b/packages/core/test/externalSpanExporterWrapper.test.ts @@ -0,0 +1,82 @@ +import { SpanKind, SpanStatusCode, TraceFlags } from "@opentelemetry/api"; +import type { ReadableSpan, SpanExporter } from "@opentelemetry/sdk-trace-node"; +import { beforeEach, describe, expect, it } from "vitest"; +import { ExternalSpanExporterWrapper } from "../src/v3/otel/tracingSDK.js"; +import { SemanticInternalAttributes } from "../src/v3/semanticInternalAttributes.js"; +import { traceContext } from "../src/v3/trace-context-api.js"; +import { StandardTraceContextManager } from "../src/v3/traceContext/manager.js"; + +const TRACEPARENT_RUN_A = "00-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-1111111111111111-01"; +const TRACEPARENT_RUN_B = "00-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-2222222222222222-01"; + +function createAttemptSpan(): ReadableSpan { + const spanCtx = { + traceId: "cccccccccccccccccccccccccccccccc", + spanId: "3333333333333333", + traceFlags: TraceFlags.SAMPLED, + }; + return { + name: "Attempt 1", + kind: SpanKind.CONSUMER, + spanContext: () => spanCtx, + parentSpanContext: undefined, + startTime: [0, 0], + endTime: [0, 0], + status: { code: SpanStatusCode.UNSET }, + attributes: { [SemanticInternalAttributes.SPAN_ATTEMPT]: true }, + links: [], + events: [], + duration: [0, 0], + ended: true, + resource: {} as any, + instrumentationLibrary: { name: "test" } as any, + droppedAttributesCount: 0, + droppedEventsCount: 0, + droppedLinksCount: 0, + } as unknown as ReadableSpan; +} + +function makeCapturingExporter(): { exporter: SpanExporter; captured: ReadableSpan[][] } { + const captured: ReadableSpan[][] = []; + const exporter: SpanExporter = { + export: (spans, cb) => { + captured.push(spans); + cb({ code: 0 } as any); + }, + shutdown: () => Promise.resolve(), + forceFlush: () => Promise.resolve(), + }; + return { exporter, captured }; +} + +describe("ExternalSpanExporterWrapper warm-start regression", () => { + let manager: StandardTraceContextManager; + + beforeEach(() => { + manager = new StandardTraceContextManager(); + traceContext.setGlobalManager(manager); + }); + + it("rewrites attempt spans using the manager's current external context, not the value captured at construction", () => { + const { exporter, captured } = makeCapturingExporter(); + + manager.traceContext = { external: { traceparent: TRACEPARENT_RUN_A } }; + + const wrapper = new ExternalSpanExporterWrapper( + exporter, + "ffffffffffffffffffffffffffffffff" + ); + + manager.traceContext = { external: { traceparent: TRACEPARENT_RUN_B } }; + + wrapper.export([createAttemptSpan()], () => {}); + + expect(captured).toHaveLength(1); + expect(captured[0]).toHaveLength(1); + + const span = captured[0]![0]!; + expect(span.parentSpanContext?.spanId).toBe("2222222222222222"); + expect(span.parentSpanContext?.spanId).not.toBe("1111111111111111"); + expect(span.spanContext().traceId).toBe("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); + }); +}); From 4dc2da5683398f0e10e340a262411e4bea3569b0 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Thu, 28 May 2026 11:28:52 +0100 Subject: [PATCH 2/2] test(core): assert parent traceId on warm-start regression test --- packages/core/test/externalSpanExporterWrapper.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/test/externalSpanExporterWrapper.test.ts b/packages/core/test/externalSpanExporterWrapper.test.ts index 3e2d3f2329a..4cff69e3aea 100644 --- a/packages/core/test/externalSpanExporterWrapper.test.ts +++ b/packages/core/test/externalSpanExporterWrapper.test.ts @@ -77,6 +77,7 @@ describe("ExternalSpanExporterWrapper warm-start regression", () => { const span = captured[0]![0]!; expect(span.parentSpanContext?.spanId).toBe("2222222222222222"); expect(span.parentSpanContext?.spanId).not.toBe("1111111111111111"); + expect(span.parentSpanContext?.traceId).toBe("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); expect(span.spanContext().traceId).toBe("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); }); });