From 51e639cd7aaa153788a803f5386a8b3a3d9abef0 Mon Sep 17 00:00:00 2001 From: Tim Fish Date: Mon, 27 Oct 2025 10:54:17 +0100 Subject: [PATCH 1/2] feat(node): Capture scope when event loop blocked --- .../suites/thread-blocked-native/isolated.mjs | 37 ++++++++++ .../suites/thread-blocked-native/test.ts | 71 +++++++++++++++++-- .../node-integration-tests/utils/index.ts | 2 +- packages/node-core/src/sdk/client.ts | 4 +- packages/node-native/package.json | 2 +- .../src/event-loop-block-integration.ts | 19 +++-- .../src/event-loop-block-watchdog.ts | 44 ++++++++++-- packages/node/src/sdk/initOtel.ts | 21 ++++-- packages/opentelemetry/src/contextManager.ts | 28 +++++++- packages/opentelemetry/src/index.ts | 1 + yarn.lock | 8 +-- 11 files changed, 206 insertions(+), 31 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/thread-blocked-native/isolated.mjs diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/isolated.mjs b/dev-packages/node-integration-tests/suites/thread-blocked-native/isolated.mjs new file mode 100644 index 000000000000..c2c0f39fc44e --- /dev/null +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/isolated.mjs @@ -0,0 +1,37 @@ +import * as Sentry from '@sentry/node'; +import { longWork } from './long-work.js'; + +setTimeout(() => { + process.exit(); +}, 10000); + +function neverResolve() { + return new Promise(() => { + // + }); +} + +const fns = [ + neverResolve, + neverResolve, + neverResolve, + neverResolve, + neverResolve, + longWork, // [5] + neverResolve, + neverResolve, + neverResolve, + neverResolve, +]; + +setTimeout(() => { + for (let id = 0; id < 10; id++) { + Sentry.withIsolationScope(async () => { + // eslint-disable-next-line no-console + console.log(`Starting task ${id}`); + Sentry.setUser({ id }); + + await fns[id](); + }); + } +}, 1000); diff --git a/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts b/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts index d168b8ce75d5..898b69ab7baa 100644 --- a/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts +++ b/dev-packages/node-integration-tests/suites/thread-blocked-native/test.ts @@ -1,6 +1,7 @@ import { join } from 'node:path'; import type { Event } from '@sentry/core'; import { afterAll, describe, expect, test } from 'vitest'; +import { NODE_VERSION } from '../../utils/index'; import { cleanupChildProcesses, createRunner } from '../../utils/runner'; function EXCEPTION(thread_id = '0', fn = 'longWork') { @@ -34,9 +35,17 @@ function EXCEPTION(thread_id = '0', fn = 'longWork') { }; } -const ANR_EVENT = { +const ANR_EVENT = (trace: boolean = false) => ({ // Ensure we have context contexts: { + ...(trace + ? { + trace: { + span_id: expect.stringMatching(/[a-f0-9]{16}/), + trace_id: expect.stringMatching(/[a-f0-9]{32}/), + }, + } + : {}), device: { arch: expect.any(String), }, @@ -63,11 +72,11 @@ const ANR_EVENT = { }, // and an exception that is our ANR exception: EXCEPTION(), -}; +}); function ANR_EVENT_WITH_DEBUG_META(file: string): Event { return { - ...ANR_EVENT, + ...ANR_EVENT(), debug_meta: { images: [ { @@ -103,7 +112,7 @@ describe('Thread Blocked Native', { timeout: 30_000 }, () => { test('Custom appRootPath', async () => { const ANR_EVENT_WITH_SPECIFIC_DEBUG_META: Event = { - ...ANR_EVENT, + ...ANR_EVENT(), debug_meta: { images: [ { @@ -134,7 +143,7 @@ describe('Thread Blocked Native', { timeout: 30_000 }, () => { test('blocked indefinitely', async () => { await createRunner(__dirname, 'indefinite.mjs') .withMockSentryServer() - .expect({ event: ANR_EVENT }) + .expect({ event: ANR_EVENT() }) .start() .completed(); }); @@ -160,7 +169,7 @@ describe('Thread Blocked Native', { timeout: 30_000 }, () => { .withMockSentryServer() .expect({ event: { - ...ANR_EVENT, + ...ANR_EVENT(), exception: EXCEPTION('0', 'longWorkOther'), }, }) @@ -179,7 +188,7 @@ describe('Thread Blocked Native', { timeout: 30_000 }, () => { expect(crashedThread).toBeDefined(); expect(event).toMatchObject({ - ...ANR_EVENT, + ...ANR_EVENT(), exception: { ...EXCEPTION(crashedThread), }, @@ -210,4 +219,52 @@ describe('Thread Blocked Native', { timeout: 30_000 }, () => { .start() .completed(); }); + + test('Capture scope via AsyncLocalStorage', async ctx => { + if (NODE_VERSION < 24) { + ctx.skip(); + return; + } + + const instrument = join(__dirname, 'instrument.mjs'); + await createRunner(__dirname, 'isolated.mjs') + .withMockSentryServer() + .withInstrument(instrument) + .expect({ + event: event => { + const crashedThread = event.threads?.values?.find(thread => thread.crashed)?.id as string; + expect(crashedThread).toBeDefined(); + + expect(event).toMatchObject({ + ...ANR_EVENT(true), + exception: { + ...EXCEPTION(crashedThread), + }, + breadcrumbs: [ + { + timestamp: expect.any(Number), + category: 'console', + data: { arguments: ['Starting task 5'], logger: 'console' }, + level: 'log', + message: 'Starting task 5', + }, + ], + user: { id: 5 }, + threads: { + values: [ + { + id: '0', + name: 'main', + crashed: true, + current: true, + main: true, + }, + ], + }, + }); + }, + }) + .start() + .completed(); + }); }); diff --git a/dev-packages/node-integration-tests/utils/index.ts b/dev-packages/node-integration-tests/utils/index.ts index e08d89a92131..92851b42ba5e 100644 --- a/dev-packages/node-integration-tests/utils/index.ts +++ b/dev-packages/node-integration-tests/utils/index.ts @@ -3,7 +3,7 @@ import { parseSemver } from '@sentry/core'; import type * as http from 'http'; import { describe } from 'vitest'; -const NODE_VERSION = parseSemver(process.versions.node).major; +export const NODE_VERSION = parseSemver(process.versions.node).major || 0; export type TestServerConfig = { url: string; diff --git a/packages/node-core/src/sdk/client.ts b/packages/node-core/src/sdk/client.ts index e631508c7392..4fde24021362 100644 --- a/packages/node-core/src/sdk/client.ts +++ b/packages/node-core/src/sdk/client.ts @@ -5,7 +5,7 @@ import { registerInstrumentations } from '@opentelemetry/instrumentation'; import type { BasicTracerProvider } from '@opentelemetry/sdk-trace-base'; import type { DynamicSamplingContext, Scope, ServerRuntimeClientOptions, TraceContext } from '@sentry/core'; import { _INTERNAL_flushLogsBuffer, applySdkMetadata, debug, SDK_VERSION, ServerRuntimeClient } from '@sentry/core'; -import { getTraceContextForScope } from '@sentry/opentelemetry'; +import { type AsyncLocalStorageLookup, getTraceContextForScope } from '@sentry/opentelemetry'; import { isMainThread, threadId } from 'worker_threads'; import { DEBUG_BUILD } from '../debug-build'; import type { NodeClientOptions } from '../types'; @@ -15,6 +15,8 @@ const DEFAULT_CLIENT_REPORT_FLUSH_INTERVAL_MS = 60_000; // 60s was chosen arbitr /** A client for using Sentry with Node & OpenTelemetry. */ export class NodeClient extends ServerRuntimeClient { public traceProvider: BasicTracerProvider | undefined; + public asyncLocalStroageLookup: AsyncLocalStorageLookup | undefined; + private _tracer: Tracer | undefined; private _clientReportInterval: NodeJS.Timeout | undefined; private _clientReportOnExitFlushListener: (() => void) | undefined; diff --git a/packages/node-native/package.json b/packages/node-native/package.json index 2a1343389ee3..8e528d351c8b 100644 --- a/packages/node-native/package.json +++ b/packages/node-native/package.json @@ -63,7 +63,7 @@ "build:tarball": "npm pack" }, "dependencies": { - "@sentry-internal/node-native-stacktrace": "^0.2.2", + "@sentry-internal/node-native-stacktrace": "^0.3.0", "@sentry/core": "10.22.0", "@sentry/node": "10.22.0" }, diff --git a/packages/node-native/src/event-loop-block-integration.ts b/packages/node-native/src/event-loop-block-integration.ts index 713093f77961..799acba15d0a 100644 --- a/packages/node-native/src/event-loop-block-integration.ts +++ b/packages/node-native/src/event-loop-block-integration.ts @@ -1,7 +1,6 @@ import { isPromise } from 'node:util/types'; import { isMainThread, Worker } from 'node:worker_threads'; import type { - Client, ClientOptions, Contexts, DsnComponents, @@ -47,7 +46,7 @@ function poll(enabled: boolean, clientOptions: ClientOptions): void { // serialized without making it a SerializedSession const session = currentSession ? { ...currentSession, toJSON: undefined } : undefined; // message the worker to tell it the main event loop is still running - threadPoll({ session, debugImages: getFilenameToDebugIdMap(clientOptions.stackParser) }, !enabled); + threadPoll(enabled, { session, debugImages: getFilenameToDebugIdMap(clientOptions.stackParser) }); } catch { // we ignore all errors } @@ -57,10 +56,17 @@ function poll(enabled: boolean, clientOptions: ClientOptions): void { * Starts polling */ function startPolling( - client: Client, + client: NodeClient, integrationOptions: Partial, ): IntegrationInternal | undefined { - registerThread(); + if (client.asyncLocalStroageLookup) { + registerThread({ + asyncLocalStorage: client.asyncLocalStroageLookup.asyncLocalStorage, + stateLookup: ['_currentContext', client.asyncLocalStroageLookup.contextSymbol], + }); + } else { + registerThread(); + } let enabled = true; @@ -161,7 +167,10 @@ const _eventLoopBlockIntegration = ((options: Partial { + polling = startPolling(client, options); + }); if (isMainThread) { await startWorker(dsn, client, options); diff --git a/packages/node-native/src/event-loop-block-watchdog.ts b/packages/node-native/src/event-loop-block-watchdog.ts index 492070a2d1dc..a4eb696c7a95 100644 --- a/packages/node-native/src/event-loop-block-watchdog.ts +++ b/packages/node-native/src/event-loop-block-watchdog.ts @@ -1,12 +1,16 @@ import { workerData } from 'node:worker_threads'; -import type { DebugImage, Event, Session, StackFrame, Thread } from '@sentry/core'; +import type { DebugImage, Event, ScopeData, Session, StackFrame, Thread } from '@sentry/core'; import { + applyScopeDataToEvent, createEventEnvelope, createSessionEnvelope, filenameIsInApp, + generateSpanId, getEnvelopeEndpointWithUrlEncodedAuth, makeSession, + mergeScopeData, normalizeUrlToBase, + Scope, stripSentryFramesAndReverse, updateSession, uuid4, @@ -16,6 +20,11 @@ import { captureStackTrace, getThreadsLastSeen } from '@sentry-internal/node-nat import type { ThreadState, WorkerStartData } from './common'; import { POLL_RATIO } from './common'; +type CurrentScopes = { + scope: Scope; + isolationScope: Scope; +}; + const { threshold, appRootPath, @@ -178,7 +187,7 @@ function applyDebugMeta(event: Event, debugImages: Record): void function getExceptionAndThreads( crashedThreadId: string, - threads: ReturnType>, + threads: ReturnType>, ): Event { const crashedThread = threads[crashedThreadId]; @@ -217,12 +226,28 @@ function getExceptionAndThreads( }; } +function applyScopeToEvent(event: Event, scope: ScopeData): void { + applyScopeDataToEvent(event, scope); + + if (!event.contexts?.trace) { + const { traceId, parentSpanId, propagationSpanId } = scope.propagationContext; + event.contexts = { + trace: { + trace_id: traceId, + span_id: propagationSpanId || generateSpanId(), + parent_span_id: parentSpanId, + }, + ...event.contexts, + }; + } +} + async function sendBlockEvent(crashedThreadId: string): Promise { if (isRateLimited()) { return; } - const threads = captureStackTrace(); + const threads = captureStackTrace(); const crashedThread = threads[crashedThreadId]; if (!crashedThread) { @@ -231,7 +256,7 @@ async function sendBlockEvent(crashedThreadId: string): Promise { } try { - await sendAbnormalSession(crashedThread.state?.session); + await sendAbnormalSession(crashedThread.pollState?.session); } catch (error) { log(`Failed to send abnormal session for thread '${crashedThreadId}':`, error); } @@ -250,8 +275,17 @@ async function sendBlockEvent(crashedThreadId: string): Promise { ...getExceptionAndThreads(crashedThreadId, threads), }; + const asyncState = threads[crashedThreadId]?.asyncState; + if (asyncState) { + // We need to rehydrate the scopes from the serialized objects so we can call getScopeData() + const scope = Object.assign(new Scope(), asyncState.scope).getScopeData(); + const isolationScope = Object.assign(new Scope(), asyncState.isolationScope).getScopeData(); + mergeScopeData(scope, isolationScope); + applyScopeToEvent(event, scope); + } + const allDebugImages: Record = Object.values(threads).reduce((acc, threadState) => { - return { ...acc, ...threadState.state?.debugImages }; + return { ...acc, ...threadState.pollState?.debugImages }; }, {}); applyDebugMeta(event, allDebugImages); diff --git a/packages/node/src/sdk/initOtel.ts b/packages/node/src/sdk/initOtel.ts index 67de29821537..ef1c8b25a2dd 100644 --- a/packages/node/src/sdk/initOtel.ts +++ b/packages/node/src/sdk/initOtel.ts @@ -14,7 +14,12 @@ import { SentryContextManager, setupOpenTelemetryLogger, } from '@sentry/node-core'; -import { SentryPropagator, SentrySampler, SentrySpanProcessor } from '@sentry/opentelemetry'; +import { + type AsyncLocalStorageLookup, + SentryPropagator, + SentrySampler, + SentrySpanProcessor, +} from '@sentry/opentelemetry'; import { DEBUG_BUILD } from '../debug-build'; import { getOpenTelemetryInstrumentationToPreload } from '../integrations/tracing'; @@ -34,8 +39,9 @@ export function initOpenTelemetry(client: NodeClient, options: AdditionalOpenTel setupOpenTelemetryLogger(); } - const provider = setupOtel(client, options); + const [provider, asyncLocalStroageLookup] = setupOtel(client, options); client.traceProvider = provider; + client.asyncLocalStroageLookup = asyncLocalStroageLookup; } interface NodePreloadOptions { @@ -82,7 +88,10 @@ function getPreloadMethods(integrationNames?: string[]): ((() => void) & { id: s } /** Just exported for tests. */ -export function setupOtel(client: NodeClient, options: AdditionalOpenTelemetryOptions = {}): BasicTracerProvider { +export function setupOtel( + client: NodeClient, + options: AdditionalOpenTelemetryOptions = {}, +): [BasicTracerProvider, AsyncLocalStorageLookup] { // Create and configure NodeTracerProvider const provider = new BasicTracerProvider({ sampler: new SentrySampler(client), @@ -106,9 +115,11 @@ export function setupOtel(client: NodeClient, options: AdditionalOpenTelemetryOp // Register as globals trace.setGlobalTracerProvider(provider); propagation.setGlobalPropagator(new SentryPropagator()); - context.setGlobalContextManager(new SentryContextManager()); - return provider; + const ctxManager = new SentryContextManager(); + context.setGlobalContextManager(ctxManager); + + return [provider, ctxManager.getAsyncLocalStorageLookup()]; } /** Just exported for tests. */ diff --git a/packages/opentelemetry/src/contextManager.ts b/packages/opentelemetry/src/contextManager.ts index e8632b095c02..ac8b2eab5c9b 100644 --- a/packages/opentelemetry/src/contextManager.ts +++ b/packages/opentelemetry/src/contextManager.ts @@ -1,3 +1,4 @@ +import type { AsyncLocalStorage } from 'node:async_hooks'; import type { Context, ContextManager } from '@opentelemetry/api'; import type { Scope } from '@sentry/core'; import { getCurrentScope, getIsolationScope } from '@sentry/core'; @@ -5,10 +6,22 @@ import { SENTRY_FORK_ISOLATION_SCOPE_CONTEXT_KEY, SENTRY_FORK_SET_ISOLATION_SCOPE_CONTEXT_KEY, SENTRY_FORK_SET_SCOPE_CONTEXT_KEY, + SENTRY_SCOPES_CONTEXT_KEY, } from './constants'; import { getScopesFromContext, setContextOnScope, setScopesOnContext } from './utils/contextData'; import { setIsSetup } from './utils/setupCheck'; +export type AsyncLocalStorageLookup = { + asyncLocalStorage: AsyncLocalStorage; + contextSymbol: symbol; +}; + +type ExtendedContextManagerInstance = new ( + ...args: unknown[] +) => ContextManagerInstance & { + getAsyncLocalStorageLookup(): AsyncLocalStorageLookup; +}; + /** * Wrap an OpenTelemetry ContextManager in a way that ensures the context is kept in sync with the Sentry Scope. * @@ -19,7 +32,7 @@ import { setIsSetup } from './utils/setupCheck'; */ export function wrapContextManagerClass( ContextManagerClass: new (...args: unknown[]) => ContextManagerInstance, -): typeof ContextManagerClass { +): ExtendedContextManagerInstance { /** * This is a custom ContextManager for OpenTelemetry, which extends the default AsyncLocalStorageContextManager. * It ensures that we create new scopes per context, so that the OTEL Context & the Sentry Scope are always in sync. @@ -69,7 +82,18 @@ export function wrapContextManagerClass; } diff --git a/packages/opentelemetry/src/index.ts b/packages/opentelemetry/src/index.ts index 6958d1c9fbdd..e0112812dc69 100644 --- a/packages/opentelemetry/src/index.ts +++ b/packages/opentelemetry/src/index.ts @@ -41,6 +41,7 @@ export { setupEventContextTrace } from './setupEventContextTrace'; export { setOpenTelemetryContextAsyncContextStrategy } from './asyncContextStrategy'; export { wrapContextManagerClass } from './contextManager'; +export type { AsyncLocalStorageLookup } from './contextManager'; export { SentryPropagator, shouldPropagateTraceForUrl } from './propagator'; export { SentrySpanProcessor } from './spanProcessor'; export { SentrySampler, wrapSamplingDecision } from './sampler'; diff --git a/yarn.lock b/yarn.lock index 06f8d3741128..cd0e775a0c65 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6936,10 +6936,10 @@ detect-libc "^2.0.3" node-abi "^3.73.0" -"@sentry-internal/node-native-stacktrace@^0.2.2": - version "0.2.2" - resolved "https://registry.yarnpkg.com/@sentry-internal/node-native-stacktrace/-/node-native-stacktrace-0.2.2.tgz#b32dde884642f100dd691b12b643361040825eeb" - integrity sha512-ZRS+a1Ik+w6awjp9na5vHBqLNkIxysfGDswLVAkjtVdBUxtfsEVI8OA6r8PijJC5Gm1oAJJap2e9H7TSiCUQIQ== +"@sentry-internal/node-native-stacktrace@^0.3.0": + version "0.3.0" + resolved "https://registry.yarnpkg.com/@sentry-internal/node-native-stacktrace/-/node-native-stacktrace-0.3.0.tgz#68c80dcf11ee070a3a54406b35d4571952caa793" + integrity sha512-ef0M2y2JDrC/H0AxMJJQInGTdZTlnwa6AAVWR4fMOpJRubkfdH2IZXE/nWU0Nj74oeJLQgdPtS6DeijLJtqq8Q== dependencies: detect-libc "^2.0.4" node-abi "^3.73.0" From 40970650fc6bd863b911f9bbebd07cf3e3999dcc Mon Sep 17 00:00:00 2001 From: Tim Fish Date: Mon, 27 Oct 2025 11:31:53 +0100 Subject: [PATCH 2/2] Typo --- packages/node-core/src/sdk/client.ts | 2 +- packages/node-native/src/event-loop-block-integration.ts | 8 +++----- packages/node/src/sdk/initOtel.ts | 4 ++-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/packages/node-core/src/sdk/client.ts b/packages/node-core/src/sdk/client.ts index 4fde24021362..8cef98ab46f1 100644 --- a/packages/node-core/src/sdk/client.ts +++ b/packages/node-core/src/sdk/client.ts @@ -15,7 +15,7 @@ const DEFAULT_CLIENT_REPORT_FLUSH_INTERVAL_MS = 60_000; // 60s was chosen arbitr /** A client for using Sentry with Node & OpenTelemetry. */ export class NodeClient extends ServerRuntimeClient { public traceProvider: BasicTracerProvider | undefined; - public asyncLocalStroageLookup: AsyncLocalStorageLookup | undefined; + public asyncLocalStorageLookup: AsyncLocalStorageLookup | undefined; private _tracer: Tracer | undefined; private _clientReportInterval: NodeJS.Timeout | undefined; diff --git a/packages/node-native/src/event-loop-block-integration.ts b/packages/node-native/src/event-loop-block-integration.ts index 799acba15d0a..b22269ed51be 100644 --- a/packages/node-native/src/event-loop-block-integration.ts +++ b/packages/node-native/src/event-loop-block-integration.ts @@ -59,11 +59,9 @@ function startPolling( client: NodeClient, integrationOptions: Partial, ): IntegrationInternal | undefined { - if (client.asyncLocalStroageLookup) { - registerThread({ - asyncLocalStorage: client.asyncLocalStroageLookup.asyncLocalStorage, - stateLookup: ['_currentContext', client.asyncLocalStroageLookup.contextSymbol], - }); + if (client.asyncLocalStorageLookup) { + const { asyncLocalStorage, contextSymbol } = client.asyncLocalStorageLookup; + registerThread({ asyncLocalStorage, stateLookup: ['_currentContext', contextSymbol] }); } else { registerThread(); } diff --git a/packages/node/src/sdk/initOtel.ts b/packages/node/src/sdk/initOtel.ts index ef1c8b25a2dd..b834bd0c13e7 100644 --- a/packages/node/src/sdk/initOtel.ts +++ b/packages/node/src/sdk/initOtel.ts @@ -39,9 +39,9 @@ export function initOpenTelemetry(client: NodeClient, options: AdditionalOpenTel setupOpenTelemetryLogger(); } - const [provider, asyncLocalStroageLookup] = setupOtel(client, options); + const [provider, asyncLocalStorageLookup] = setupOtel(client, options); client.traceProvider = provider; - client.asyncLocalStroageLookup = asyncLocalStroageLookup; + client.asyncLocalStorageLookup = asyncLocalStorageLookup; } interface NodePreloadOptions {