From a89edfc1a9e216769314d8ea1dcbf60343acaa33 Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Thu, 16 Oct 2025 17:02:43 +0200 Subject: [PATCH 01/12] feat(core): Instrument LangChain AI --- .../node-integration-tests/package.json | 2 + .../tracing/langchain/instrument-with-pii.mjs | 17 + .../suites/tracing/langchain/instrument.mjs | 17 + .../suites/tracing/langchain/scenario.mjs | 108 +++++ .../suites/tracing/langchain/test.ts | 163 ++++++++ packages/astro/src/index.server.ts | 1 + packages/aws-serverless/src/index.ts | 1 + packages/bun/src/index.ts | 1 + packages/core/src/index.ts | 11 + .../core/src/utils/langchain/constants.ts | 24 ++ packages/core/src/utils/langchain/index.ts | 341 ++++++++++++++++ packages/core/src/utils/langchain/types.ts | 124 ++++++ packages/core/src/utils/langchain/utils.ts | 383 ++++++++++++++++++ packages/google-cloud-serverless/src/index.ts | 1 + packages/node/src/index.ts | 2 + .../node/src/integrations/tracing/index.ts | 3 + .../integrations/tracing/langchain/index.ts | 109 +++++ .../tracing/langchain/instrumentation.ts | 201 +++++++++ yarn.lock | 104 ++++- 19 files changed, 1608 insertions(+), 5 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/langchain/instrument-with-pii.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/langchain/instrument.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/langchain/scenario.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/langchain/test.ts create mode 100644 packages/core/src/utils/langchain/constants.ts create mode 100644 packages/core/src/utils/langchain/index.ts create mode 100644 packages/core/src/utils/langchain/types.ts create mode 100644 packages/core/src/utils/langchain/utils.ts create mode 100644 packages/node/src/integrations/tracing/langchain/index.ts create mode 100644 packages/node/src/integrations/tracing/langchain/instrumentation.ts diff --git a/dev-packages/node-integration-tests/package.json b/dev-packages/node-integration-tests/package.json index 5d7a02f7327c..a693f4806fb0 100644 --- a/dev-packages/node-integration-tests/package.json +++ b/dev-packages/node-integration-tests/package.json @@ -27,6 +27,8 @@ "@aws-sdk/client-s3": "^3.552.0", "@google/genai": "^1.20.0", "@growthbook/growthbook": "^1.6.1", + "@langchain/anthropic": "^0.3.10", + "@langchain/core": "^0.3.28", "@hapi/hapi": "^21.3.10", "@hono/node-server": "^1.19.4", "@nestjs/common": "^11", diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-with-pii.mjs new file mode 100644 index 000000000000..cb68a6f7683e --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-with-pii.mjs @@ -0,0 +1,17 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + sendDefaultPii: true, + transport: loggingTransport, + beforeSendTransaction: event => { + // Filter out mock express server transactions + if (event.transaction.includes('/v1/messages')) { + return null; + } + return event; + }, +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/instrument.mjs new file mode 100644 index 000000000000..b4ce44f3e91a --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/instrument.mjs @@ -0,0 +1,17 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + sendDefaultPii: false, + transport: loggingTransport, + beforeSendTransaction: event => { + // Filter out mock express server transactions + if (event.transaction.includes('/v1/messages')) { + return null; + } + return event; + }, +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/scenario.mjs new file mode 100644 index 000000000000..132e519c0b8c --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/scenario.mjs @@ -0,0 +1,108 @@ +import { ChatAnthropic } from '@langchain/anthropic'; +import * as Sentry from '@sentry/node'; +import express from 'express'; + +function startMockAnthropicServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1/messages', (req, res) => { + const model = req.body.model; + + if (model === 'error-model') { + res + .status(400) + .set('request-id', 'mock-request-123') + .json({ + type: 'error', + error: { + type: 'invalid_request_error', + message: 'Model not found', + }, + }); + return; + } + + // Simulate basic response + res.json({ + id: 'msg_test123', + type: 'message', + role: 'assistant', + content: [ + { + type: 'text', + text: 'Mock response from Anthropic!', + }, + ], + model: model, + stop_reason: 'end_turn', + stop_sequence: null, + usage: { + input_tokens: 10, + output_tokens: 15, + }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockAnthropicServer(); + const baseUrl = `http://localhost:${server.address().port}`; + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + // Test 1: Basic chat model invocation + const model1 = new ChatAnthropic({ + model: 'claude-3-5-sonnet-20241022', + temperature: 0.7, + maxTokens: 100, + apiKey: 'mock-api-key', + clientOptions: { + baseURL: baseUrl, + }, + }); + + await model1.invoke('Tell me a joke'); + + // Test 2: Chat with different model + const model2 = new ChatAnthropic({ + model: 'claude-3-opus-20240229', + temperature: 0.9, + topP: 0.95, + maxTokens: 200, + apiKey: 'mock-api-key', + clientOptions: { + baseURL: baseUrl, + }, + }); + + await model2.invoke([ + { role: 'system', content: 'You are a helpful assistant' }, + { role: 'user', content: 'What is the capital of France?' }, + ]); + + // Test 3: Error handling + const errorModel = new ChatAnthropic({ + model: 'error-model', + apiKey: 'mock-api-key', + clientOptions: { + baseURL: baseUrl, + }, + }); + + try { + await errorModel.invoke('This will fail'); + } catch (error) { + // Expected error + } + }); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts new file mode 100644 index 000000000000..7bd9c416e5fa --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts @@ -0,0 +1,163 @@ +import { afterAll, describe, expect } from 'vitest'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; + +describe('LangChain integration', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE = { + transaction: 'main', + spans: expect.arrayContaining([ + // First span - chat model with claude-3-5-sonnet + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'auto.ai.langchain', + 'gen_ai.system': '"anthropic"', + 'gen_ai.request.model': '"claude-3-5-sonnet-20241022"', + 'gen_ai.request.temperature': 0.7, + 'gen_ai.request.max_tokens': 100, + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 15, + 'gen_ai.usage.total_tokens': 25, + 'gen_ai.response.id': expect.any(String), + 'gen_ai.response.model': expect.any(String), + 'gen_ai.response.stop_reason': expect.any(String), + }), + description: 'chat "claude-3-5-sonnet-20241022"', + op: 'gen_ai.chat', + origin: 'auto.ai.langchain', + status: 'ok', + }), + // Second span - chat model with claude-3-opus + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'auto.ai.langchain', + 'gen_ai.system': '"anthropic"', + 'gen_ai.request.model': '"claude-3-opus-20240229"', + 'gen_ai.request.temperature': 0.9, + 'gen_ai.request.top_p': 0.95, + 'gen_ai.request.max_tokens': 200, + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 15, + 'gen_ai.usage.total_tokens': 25, + 'gen_ai.response.id': expect.any(String), + 'gen_ai.response.model': expect.any(String), + 'gen_ai.response.stop_reason': expect.any(String), + }), + description: 'chat "claude-3-opus-20240229"', + op: 'gen_ai.chat', + origin: 'auto.ai.langchain', + status: 'ok', + }), + // Third span - error handling + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'auto.ai.langchain', + 'gen_ai.system': '"anthropic"', + 'gen_ai.request.model': '"error-model"', + }), + description: 'chat "error-model"', + op: 'gen_ai.chat', + origin: 'auto.ai.langchain', + status: 'unknown_error', + }), + ]), + }; + + const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE = { + transaction: 'main', + spans: expect.arrayContaining([ + // First span - chat model with PII + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'auto.ai.langchain', + 'gen_ai.system': '"anthropic"', + 'gen_ai.request.model': '"claude-3-5-sonnet-20241022"', + 'gen_ai.request.temperature': 0.7, + 'gen_ai.request.max_tokens': 100, + 'gen_ai.request.messages': expect.any(String), // Should include messages when recordInputs: true + 'gen_ai.response.text': expect.any(String), // Should include response when recordOutputs: true + 'gen_ai.response.id': expect.any(String), + 'gen_ai.response.model': expect.any(String), + 'gen_ai.response.stop_reason': expect.any(String), + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 15, + 'gen_ai.usage.total_tokens': 25, + }), + description: 'chat "claude-3-5-sonnet-20241022"', + op: 'gen_ai.chat', + origin: 'auto.ai.langchain', + status: 'ok', + }), + // Second span - chat model with PII + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'auto.ai.langchain', + 'gen_ai.system': '"anthropic"', + 'gen_ai.request.model': '"claude-3-opus-20240229"', + 'gen_ai.request.temperature': 0.9, + 'gen_ai.request.top_p': 0.95, + 'gen_ai.request.max_tokens': 200, + 'gen_ai.request.messages': expect.any(String), // Should include messages when recordInputs: true + 'gen_ai.response.text': expect.any(String), // Should include response when recordOutputs: true + 'gen_ai.response.id': expect.any(String), + 'gen_ai.response.model': expect.any(String), + 'gen_ai.response.stop_reason': expect.any(String), + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 15, + 'gen_ai.usage.total_tokens': 25, + }), + description: 'chat "claude-3-opus-20240229"', + op: 'gen_ai.chat', + origin: 'auto.ai.langchain', + status: 'ok', + }), + // Third span - error handling with PII + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'auto.ai.langchain', + 'gen_ai.system': '"anthropic"', + 'gen_ai.request.model': '"error-model"', + 'gen_ai.request.messages': expect.any(String), // Should include messages when recordInputs: true + }), + description: 'chat "error-model"', + op: 'gen_ai.chat', + origin: 'auto.ai.langchain', + status: 'unknown_error', + }), + ]), + }; + + createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument.mjs', (createRunner, test) => { + test('creates langchain related spans with sendDefaultPii: false', async () => { + await createRunner() + .ignore('event') + .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE }) + .start() + .completed(); + }); + }); + + createEsmAndCjsTests(__dirname, 'scenario.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { + test('creates langchain related spans with sendDefaultPii: true', async () => { + await createRunner() + .ignore('event') + .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE }) + .start() + .completed(); + }); + }); +}); diff --git a/packages/astro/src/index.server.ts b/packages/astro/src/index.server.ts index 15158bdbb7bc..b617f5010a3d 100644 --- a/packages/astro/src/index.server.ts +++ b/packages/astro/src/index.server.ts @@ -93,6 +93,7 @@ export { onUncaughtExceptionIntegration, onUnhandledRejectionIntegration, openAIIntegration, + langChainIntegration, parameterize, pinoIntegration, postgresIntegration, diff --git a/packages/aws-serverless/src/index.ts b/packages/aws-serverless/src/index.ts index 5ff30f069486..ed66b3c99ee0 100644 --- a/packages/aws-serverless/src/index.ts +++ b/packages/aws-serverless/src/index.ts @@ -56,6 +56,7 @@ export { onUncaughtExceptionIntegration, onUnhandledRejectionIntegration, openAIIntegration, + langChainIntegration, modulesIntegration, contextLinesIntegration, nodeContextIntegration, diff --git a/packages/bun/src/index.ts b/packages/bun/src/index.ts index 5ec1568229e4..4c503c1fb6c1 100644 --- a/packages/bun/src/index.ts +++ b/packages/bun/src/index.ts @@ -76,6 +76,7 @@ export { onUncaughtExceptionIntegration, onUnhandledRejectionIntegration, openAIIntegration, + langChainIntegration, modulesIntegration, contextLinesIntegration, nodeContextIntegration, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 7a6c5c2e17d3..2d2ea817cd92 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -144,6 +144,8 @@ export { ANTHROPIC_AI_INTEGRATION_NAME } from './utils/anthropic-ai/constants'; export { instrumentGoogleGenAIClient } from './utils/google-genai'; export { GOOGLE_GENAI_INTEGRATION_NAME } from './utils/google-genai/constants'; export type { GoogleGenAIResponse } from './utils/google-genai/types'; +export { createLangChainCallbackHandler } from './utils/langchain'; +export { LANGCHAIN_INTEGRATION_NAME } from './utils/langchain/constants'; export type { OpenAiClient, OpenAiOptions, InstrumentedMethod } from './utils/openai/types'; export type { AnthropicAiClient, @@ -157,6 +159,15 @@ export type { GoogleGenAIOptions, GoogleGenAIIstrumentedMethod, } from './utils/google-genai/types'; +export type { + LangChainOptions, + LangChainIntegration, + LangChainSerializedLLM, + LangChainMessage, + LangChainLLMResult, + LangChainTool, + LangChainDocument, +} from './utils/langchain/types'; export type { FeatureFlag } from './utils/featureFlags'; export { diff --git a/packages/core/src/utils/langchain/constants.ts b/packages/core/src/utils/langchain/constants.ts new file mode 100644 index 000000000000..f15476b41ae4 --- /dev/null +++ b/packages/core/src/utils/langchain/constants.ts @@ -0,0 +1,24 @@ +export const LANGCHAIN_INTEGRATION_NAME = 'LangChain'; +export const LANGCHAIN_ORIGIN = 'auto.ai.langchain'; + +/** + * LangChain event types we instrument + * Based on LangChain.js callback system + * @see https://js.langchain.com/docs/concepts/callbacks/ + */ +export const LANGCHAIN_EVENT_TYPES = { + CHAT_MODEL_START: 'handleChatModelStart', + LLM_START: 'handleLLMStart', + LLM_NEW_TOKEN: 'handleLLMNewToken', + LLM_END: 'handleLLMEnd', + LLM_ERROR: 'handleLLMError', + CHAIN_START: 'handleChainStart', + CHAIN_END: 'handleChainEnd', + CHAIN_ERROR: 'handleChainError', + TOOL_START: 'handleToolStart', + TOOL_END: 'handleToolEnd', + TOOL_ERROR: 'handleToolError', + RETRIEVER_START: 'handleRetrieverStart', + RETRIEVER_END: 'handleRetrieverEnd', + RETRIEVER_ERROR: 'handleRetrieverError', +} as const; diff --git a/packages/core/src/utils/langchain/index.ts b/packages/core/src/utils/langchain/index.ts new file mode 100644 index 000000000000..1130bd65ae8e --- /dev/null +++ b/packages/core/src/utils/langchain/index.ts @@ -0,0 +1,341 @@ +import { captureException } from '../../exports'; +import { SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '../../semanticAttributes'; +import { SPAN_STATUS_ERROR } from '../../tracing'; +import { startSpanManual } from '../../tracing/trace'; +import type { Span, SpanAttributeValue } from '../../types-hoist/span'; +import { GEN_AI_OPERATION_NAME_ATTRIBUTE, GEN_AI_REQUEST_MODEL_ATTRIBUTE } from '../ai/gen-ai-attributes'; +import { LANGCHAIN_ORIGIN } from './constants'; +import type { + LangChainLLMResult, + LangChainMessage, + LangChainOptions, + LangChainRunId, + LangChainSerializedLLM, +} from './types'; +import { + addLLMResponseAttributes, + extractChatModelRequestAttributes, + extractLLMRequestAttributes, + getInvocationParams, +} from './utils'; + +/** + * Creates a Sentry callback handler for LangChain + * Returns a plain object that LangChain will call via duck-typing + * + * This is a stateful handler that tracks spans across multiple LangChain executions. + */ +export function createLangChainCallbackHandler(options: LangChainOptions = {}): { + handleLLMStart?: ( + llm: LangChainSerializedLLM, + prompts: string[], + runId: LangChainRunId, + parentRunId?: LangChainRunId, + extraParams?: Record, + tags?: string[] | Record, + metadata?: Record, + runName?: string | Record, + ) => void; + handleChatModelStart?: ( + llm: LangChainSerializedLLM, + messages: LangChainMessage[][], + runId: LangChainRunId, + parentRunId?: LangChainRunId, + extraParams?: Record, + tags?: string[] | Record, + metadata?: Record, + runName?: string | Record, + ) => void; + handleLLMNewToken?: (token: string, runId: LangChainRunId) => void; + handleLLMEnd?: (output: LangChainLLMResult, runId: LangChainRunId) => void; + handleLLMError?: (error: Error, runId: LangChainRunId) => void; + handleChainStart?: ( + chain: { name?: string }, + inputs: Record, + runId: LangChainRunId, + parentRunId?: LangChainRunId, + ) => void; + handleChainEnd?: (outputs: Record, runId: LangChainRunId) => void; + handleChainError?: (error: Error, runId: LangChainRunId) => void; + handleToolStart?: ( + tool: { name?: string }, + input: string, + runId: LangChainRunId, + parentRunId?: LangChainRunId, + ) => void; + handleToolEnd?: (output: string, runId: LangChainRunId) => void; + handleToolError?: (error: Error, runId: LangChainRunId) => void; +} { + const recordInputs = options.recordInputs ?? false; + const recordOutputs = options.recordOutputs ?? false; + + // Internal state - single instance tracks all spans + const spanMap = new Map(); + + /** + * Exit a span and clean up + */ + const exitSpan = (runId: LangChainRunId): void => { + const span = spanMap.get(runId); + if (span) { + span.end(); + spanMap.delete(runId); + } + }; + + /** + * Handler for LLM Start + * This handler will be called by LangChain's callback handler when an LLM event is detected. + */ + const handler = { + handleLLMStart( + llm: LangChainSerializedLLM, + prompts: string[], + runId: LangChainRunId, + _parentRunId?: LangChainRunId, + _extraParams?: Record, + tags?: string[] | Record, + metadata?: Record, + _runName?: string | Record, + ) { + try { + const invocationParams = getInvocationParams(tags); + const attributes = extractLLMRequestAttributes(llm, prompts, recordInputs, invocationParams, metadata); + const modelName = attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]; + const operationName = attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]; + + startSpanManual( + { + name: `${operationName} ${modelName}`, + op: 'gen_ai.pipeline', + attributes, + }, + span => { + spanMap.set(runId, span); + return span; + }, + ); + } catch { + // Silently ignore errors, llm errors are captured by the handleLLMError handler + } + }, + + // Chat Model Start Handler + handleChatModelStart( + llm: LangChainSerializedLLM, + messages: LangChainMessage[][], + runId: LangChainRunId, + _parentRunId?: LangChainRunId, + _extraParams?: Record, + tags?: string[] | Record, + metadata?: Record, + _runName?: string | Record, + ) { + try { + const invocationParams = getInvocationParams(tags); + const attributes = extractChatModelRequestAttributes(llm, messages, recordInputs, invocationParams, metadata); + const modelName = attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]; + const operationName = attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]; + + startSpanManual( + { + name: `${operationName} ${modelName}`, + op: 'gen_ai.chat', + attributes, + }, + span => { + spanMap.set(runId, span); + + return span; + }, + ); + } catch { + // Silently ignore errors, chat model start errors are captured by the handleChatModelError handler + } + }, + + // LLM End Handler - note: handleLLMEnd with capital LLM (used by both LLMs and chat models!) + handleLLMEnd( + output: LangChainLLMResult, + runId: LangChainRunId, + _parentRunId?: string, + _tags?: string[], + _extraParams?: Record, + ) { + try { + const span = spanMap.get(runId); + if (span) { + addLLMResponseAttributes(span, output, recordOutputs); + exitSpan(runId); + } + } catch { + // Silently ignore errors, llm end errors are captured by the handleLLMError handler + } + }, + + // LLM Error Handler - note: handleLLMError with capital LLM + handleLLMError(error: Error, runId: LangChainRunId) { + try { + const span = spanMap.get(runId); + if (span) { + span.setStatus({ code: SPAN_STATUS_ERROR, message: 'llm_error' }); + exitSpan(runId); + } + + captureException(error, { + mechanism: { + handled: false, + type: LANGCHAIN_ORIGIN, + data: { handler: 'handleLLMError' }, + }, + }); + } catch { + // silently ignore errors + } + }, + + // Chain Start Handler + handleChainStart( + chain: { name?: string }, + inputs: Record, + runId: LangChainRunId, + _parentRunId?: LangChainRunId, + ) { + try { + const chainName = chain.name || 'unknown_chain'; + const attributes: Record = { + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', + 'langchain.chain.name': chainName, + }; + + // Add inputs if recordInputs is enabled + if (recordInputs) { + attributes['langchain.chain.inputs'] = JSON.stringify(inputs); + } + + startSpanManual( + { + name: `chain ${chainName}`, + op: 'gen_ai.invoke_agent', + attributes, + }, + span => { + spanMap.set(runId, span); + + return span; + }, + ); + } catch { + // Silently ignore errors, chain start errors are captured by the handleChainError handler + } + }, + + // Chain End Handler + handleChainEnd(outputs: Record, runId: LangChainRunId) { + try { + const span = spanMap.get(runId); + if (span) { + // Add outputs if recordOutputs is enabled + if (recordOutputs) { + span.setAttributes({ + 'langchain.chain.outputs': JSON.stringify(outputs), + }); + } + exitSpan(runId); + } + } catch { + // Silently ignore errors, chain end errors are captured by the handleChainError handler + } + }, + + // Chain Error Handler + handleChainError(error: Error, runId: LangChainRunId) { + try { + const span = spanMap.get(runId); + if (span) { + span.setStatus({ code: SPAN_STATUS_ERROR, message: 'chain_error' }); + exitSpan(runId); + } + + captureException(error, { + mechanism: { + handled: false, + type: LANGCHAIN_ORIGIN, + }, + }); + } catch { + // silently ignore errors + } + }, + + // Tool Start Handler + handleToolStart(tool: { name?: string }, input: string, runId: LangChainRunId, _parentRunId?: LangChainRunId) { + try { + const toolName = tool.name || 'unknown_tool'; + const attributes: Record = { + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: LANGCHAIN_ORIGIN, + 'gen_ai.tool.name': toolName, + }; + + // Add input if recordInputs is enabled + if (recordInputs) { + attributes['gen_ai.tool.input'] = input; + } + + startSpanManual( + { + name: `execute_tool ${toolName}`, + op: 'gen_ai.execute_tool', + attributes, + }, + span => { + spanMap.set(runId, span); + return span; + }, + ); + } catch { + // Silently ignore errors, tool start errors are captured by the handleToolError handler + } + }, + + // Tool End Handler + handleToolEnd(output: string, runId: LangChainRunId) { + try { + const span = spanMap.get(runId); + if (span) { + // Add output if recordOutputs is enabled + if (recordOutputs) { + span.setAttributes({ + 'gen_ai.tool.output': output, + }); + } + exitSpan(runId); + } + } catch { + // Silently ignore errors, tool start errors are captured by the handleToolError handler + } + }, + + // Tool Error Handler + handleToolError(error: Error, runId: LangChainRunId) { + try { + const span = spanMap.get(runId); + if (span) { + span.setStatus({ code: SPAN_STATUS_ERROR, message: 'tool_error' }); + exitSpan(runId); + } + + captureException(error, { + mechanism: { + handled: false, + type: LANGCHAIN_ORIGIN, + }, + }); + } catch { + // silently ignore errors + } + }, + }; + + return handler; +} diff --git a/packages/core/src/utils/langchain/types.ts b/packages/core/src/utils/langchain/types.ts new file mode 100644 index 000000000000..4e18b236d630 --- /dev/null +++ b/packages/core/src/utils/langchain/types.ts @@ -0,0 +1,124 @@ +/** + * Options for LangChain integration + */ +export interface LangChainOptions { + /** + * Whether to record input messages/prompts + * @default false (respects sendDefaultPii option) + */ + recordInputs?: boolean; + + /** + * Whether to record output text and responses + * @default false (respects sendDefaultPii option) + */ + recordOutputs?: boolean; +} + +/** + * LangChain LLM/Chat Model serialized data + */ +export interface LangChainSerializedLLM { + [key: string]: unknown; + type?: string; + lc?: number; + id?: string[]; + kwargs?: { + [key: string]: unknown; + model?: string; + temperature?: number; + }; +} + +/** + * LangChain message structure + * Supports both regular messages and LangChain serialized format + */ +export interface LangChainMessage { + [key: string]: unknown; + // Regular message format + type?: string; + content?: string; + message?: { + content?: unknown[]; + type?: string; + }; + role?: string; + additional_kwargs?: Record; + // LangChain serialized format + lc?: number; + id?: string[]; + kwargs?: { + [key: string]: unknown; + content?: string; + additional_kwargs?: Record; + response_metadata?: Record; + }; +} + +/** + * LangChain LLM result structure + */ +export interface LangChainLLMResult { + [key: string]: unknown; + generations: Array<{ + text?: string; + message?: LangChainMessage; + generation_info?: { + [key: string]: unknown; + + finish_reason?: string; + logprobs?: unknown; + }; + }>; + llmOutput?: { + [key: string]: unknown; + tokenUsage?: { + completionTokens?: number; + promptTokens?: number; + totalTokens?: number; + }; + model_name?: string; + }; +} + +/** + * LangChain Run ID + */ +export type LangChainRunId = string; + +/** + * LangChain span context stored per run + */ +export interface LangChainSpanContext { + spanId: string; + traceId?: string; + parentSpanId?: string; + streamingBuffer?: string[]; +} + +/** + * LangChain Tool structure + */ +export interface LangChainTool { + [key: string]: unknown; + name: string; + description?: string; +} + +/** + * LangChain Document structure for retrievers + */ +export interface LangChainDocument { + [key: string]: unknown; + pageContent: string; + metadata?: Record; +} + +/** + * Integration interface for type safety + */ +export interface LangChainIntegration { + name: string; + options: LangChainOptions; +} diff --git a/packages/core/src/utils/langchain/utils.ts b/packages/core/src/utils/langchain/utils.ts new file mode 100644 index 000000000000..0f9fb317c836 --- /dev/null +++ b/packages/core/src/utils/langchain/utils.ts @@ -0,0 +1,383 @@ +import type { Span } from '../..'; +import { type SpanAttributeValue, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '../..'; +import { + GEN_AI_OPERATION_NAME_ATTRIBUTE, + GEN_AI_REQUEST_FREQUENCY_PENALTY_ATTRIBUTE, + GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE, + GEN_AI_REQUEST_MESSAGES_ATTRIBUTE, + GEN_AI_REQUEST_MODEL_ATTRIBUTE, + GEN_AI_REQUEST_PRESENCE_PENALTY_ATTRIBUTE, + GEN_AI_REQUEST_STREAM_ATTRIBUTE, + GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE, + GEN_AI_REQUEST_TOP_P_ATTRIBUTE, + GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE, + GEN_AI_RESPONSE_MODEL_ATTRIBUTE, + GEN_AI_RESPONSE_TEXT_ATTRIBUTE, + GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE, + GEN_AI_SYSTEM_ATTRIBUTE, + GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE, + GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, + GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, +} from '../ai/gen-ai-attributes'; +import { LANGCHAIN_ORIGIN } from './constants'; +import type { LangChainLLMResult, LangChainMessage, LangChainSerializedLLM } from './types'; + +/** + * Extract invocation params from tags object + * LangChain passes runtime parameters in the tags object + */ +export function getInvocationParams(tags?: string[] | Record): Record | undefined { + if (!tags || Array.isArray(tags)) { + return undefined; + } + return tags.invocation_params as Record | undefined; +} + +/** + * Normalize a single message role to standard gen_ai format + */ +export function normalizeMessageRole(role: string): string { + const roleMap: Record = { + human: 'user', + ai: 'assistant', + system: 'system', + function: 'function', + tool: 'tool', + }; + + const normalizedRole = role.toLowerCase(); + return roleMap[normalizedRole] || normalizedRole; +} + +/** + * Extract role from constructor name + */ +export function normalizeRoleName(roleName: string): string { + if (roleName.includes('System')) { + return 'system'; + } + if (roleName.includes('Human')) { + return 'user'; + } + if (roleName.includes('AI') || roleName.includes('Assistant')) { + return 'assistant'; + } + if (roleName.includes('Function')) { + return 'function'; + } + if (roleName.includes('Tool')) { + return 'tool'; + } + return 'user'; +} + +/** + * Normalize LangChain messages to simple {role, content} format + * Handles both raw message objects and serialized LangChain message format + */ +export function normalizeLangChainMessages(messages: LangChainMessage[]): Array<{ role: string; content: string }> { + return messages.map(message => { + // First, try to get the message type from _getType() method (most reliable) + if (typeof (message as { _getType?: () => string })._getType === 'function') { + const messageType = (message as { _getType: () => string })._getType(); + return { + role: normalizeMessageRole(messageType), + content: String((message as { content?: string }).content || ''), + }; + } + + // Check constructor name (for LangChain message objects like SystemMessage, HumanMessage, etc.) + const constructorName = message.constructor?.name; + if (constructorName) { + const role = normalizeRoleName(constructorName); + return { + role: normalizeMessageRole(role), + content: String((message as { content?: string }).content || ''), + }; + } + + // Handle message objects with type field + if (message.type) { + const role = String(message.type).toLowerCase(); + return { + role: normalizeMessageRole(role), + content: String(message.content || ''), + }; + } + + // Handle regular message objects with role and content + if (message.role && message.content) { + return { + role: normalizeMessageRole(String(message.role)), + content: String(message.content), + }; + } + + // Handle LangChain serialized format with lc: 1 + if (message.lc === 1 && message.kwargs) { + // Extract role from the message type (e.g., HumanMessage -> user) + const messageType = Array.isArray(message.id) && message.id.length > 0 ? message.id[message.id.length - 1] : ''; + const role = typeof messageType === 'string' ? normalizeRoleName(messageType) : 'user'; + + return { + role: normalizeMessageRole(role), + content: String(message.kwargs.content || ''), + }; + } + + // Fallback: return as-is if we can't normalize + return { + role: 'user', + content: String(message.content || JSON.stringify(message)), + }; + }); +} + +/** + * Extract common request attributes shared by LLM and chat model requests + */ +export function extractCommonRequestAttributes( + serialized: LangChainSerializedLLM, + invocationParams?: Record, + langSmithMetadata?: Record, +): Record { + const attributes: Record = {}; + + // Priority: invocationParams > LangSmith > kwargs + const temperature = + invocationParams?.temperature ?? langSmithMetadata?.ls_temperature ?? serialized.kwargs?.temperature; + if (Number(temperature)) { + attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE] = Number(temperature); + } + + const maxTokens = invocationParams?.max_tokens ?? langSmithMetadata?.ls_max_tokens ?? serialized.kwargs?.max_tokens; + if (Number(maxTokens)) { + attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE] = Number(maxTokens); + } + + const topP = invocationParams?.top_p ?? serialized.kwargs?.top_p; + if (Number(topP)) { + attributes[GEN_AI_REQUEST_TOP_P_ATTRIBUTE] = Number(topP); + } + + const frequencyPenalty = invocationParams?.frequency_penalty ?? serialized.kwargs?.frequency_penalty; + if (Number(frequencyPenalty)) { + attributes[GEN_AI_REQUEST_FREQUENCY_PENALTY_ATTRIBUTE] = Number(frequencyPenalty); + } + + const presencePenalty = invocationParams?.presence_penalty ?? serialized.kwargs?.presence_penalty; + if (Number(presencePenalty)) { + attributes[GEN_AI_REQUEST_PRESENCE_PENALTY_ATTRIBUTE] = Number(presencePenalty); + } + + const streaming = invocationParams?.stream; + if (streaming !== undefined && streaming !== null) { + // Sometimes this is set to false even for stream requests + // This issue stems from LangChain's callback handler and should be investigated + attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] = Boolean(streaming); + } + + return attributes; +} + +/** + * Extract request attributes from LLM start event + */ +export function extractLLMRequestAttributes( + serialized: LangChainSerializedLLM, + prompts: string[], + recordInputs: boolean, + invocationParams?: Record, + langSmithMetadata?: Record, +): Record { + const system = JSON.stringify(langSmithMetadata?.ls_provider); + const modelName = JSON.stringify(invocationParams?.model ?? langSmithMetadata?.ls_model_name ?? 'unknown'); + + const attributes: Record = { + [GEN_AI_SYSTEM_ATTRIBUTE]: system, + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'pipeline', + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: modelName, + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: LANGCHAIN_ORIGIN, + ...extractCommonRequestAttributes(serialized, invocationParams, langSmithMetadata), + }; + + // Add prompts if recordInputs is enabled + if (recordInputs && prompts && prompts.length > 0) { + // Convert string prompts to message format + const messages = prompts.map(prompt => ({ role: 'user', content: prompt })); + attributes[GEN_AI_REQUEST_MESSAGES_ATTRIBUTE] = JSON.stringify(messages); + } + + return attributes; +} + +/** + * Extract request attributes from chat model start event + */ +export function extractChatModelRequestAttributes( + serialized: LangChainSerializedLLM, + messages: LangChainMessage[][], + recordInputs: boolean, + invocationParams?: Record, + langSmithMetadata?: Record, +): Record { + // Provider either exists in LangSmith metadata or is the 3rd index of the id array of the serialized LLM + const system = JSON.stringify(langSmithMetadata?.ls_provider ?? serialized.id?.[2]); + const modelName = JSON.stringify(invocationParams?.model ?? langSmithMetadata?.ls_model_name ?? 'unknown'); + + const attributes: Record = { + [GEN_AI_SYSTEM_ATTRIBUTE]: system, + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: modelName, + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: LANGCHAIN_ORIGIN, + ...extractCommonRequestAttributes(serialized, invocationParams, langSmithMetadata), + }; + + // Add messages if recordInputs is enabled + if (recordInputs && messages && messages.length > 0) { + // Flatten the messages array (LangChain passes array of message arrays) + const flatMessages = messages.flat(); + // Normalize messages to extract content from LangChain serialized format + const normalizedMessages = normalizeLangChainMessages(flatMessages); + attributes[GEN_AI_REQUEST_MESSAGES_ATTRIBUTE] = JSON.stringify(normalizedMessages); + } + + return attributes; +} + +/** + * Extract tool calls attributes from generations + */ +export function extractToolCallsAttributes(generations: LangChainMessage[][], span: Span): void { + const toolCalls: Array = []; + + // Flatten the generations array (LangChain returns [[generation]]) + const flatGenerations = generations.flat(); + + for (const gen of flatGenerations) { + // Check if message has a content array (Anthropic format) + if (gen.message?.content && Array.isArray(gen.message.content)) { + for (const contentItem of gen.message.content) { + const typedContent = contentItem as { type?: string; id?: string; name?: string; input?: unknown }; + if (typedContent.type === 'tool_use') { + toolCalls.push(typedContent); + } + } + } + } + + if (toolCalls.length > 0) { + span.setAttributes({ + [GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]: JSON.stringify(toolCalls), + }); + } +} + +/** + * Extract token usage attributes from LLM output + */ +export function extractTokenUsageAttributes(llmOutput: LangChainLLMResult['llmOutput'], span: Span): void { + if (!llmOutput) return; + // Try standard tokenUsage format (OpenAI) + const tokenUsage = llmOutput.tokenUsage as + | { promptTokens?: number; completionTokens?: number; totalTokens?: number } + | undefined; + // Try Anthropic format (usage.input_tokens, etc.) + const anthropicUsage = llmOutput.usage as + | { + input_tokens?: number; + output_tokens?: number; + cache_creation_input_tokens?: number; + cache_read_input_tokens?: number; + } + | undefined; + + if (tokenUsage) { + if (Number(tokenUsage.promptTokens)) { + span.setAttributes({ [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: tokenUsage.promptTokens }); + } + if (Number(tokenUsage.completionTokens)) { + span.setAttributes({ [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: tokenUsage.completionTokens }); + } + if (Number(tokenUsage.totalTokens)) { + span.setAttributes({ [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: tokenUsage.totalTokens }); + } + } else if (anthropicUsage) { + // Handle Anthropic format + if (Number(anthropicUsage.input_tokens)) { + span.setAttributes({ [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: anthropicUsage.input_tokens }); + } + if (Number(anthropicUsage.output_tokens)) { + span.setAttributes({ [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: anthropicUsage.output_tokens }); + } + // Calculate total tokens for Anthropic + const total = (anthropicUsage.input_tokens || 0) + (anthropicUsage.output_tokens || 0); + if (total > 0) { + span.setAttributes({ [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: total }); + } + // Add cache tokens if present + if (anthropicUsage.cache_creation_input_tokens !== undefined) { + span.setAttributes({ 'gen_ai.usage.cache_creation_input_tokens': anthropicUsage.cache_creation_input_tokens }); + } + if (anthropicUsage.cache_read_input_tokens !== undefined) { + span.setAttributes({ 'gen_ai.usage.cache_read_input_tokens': anthropicUsage.cache_read_input_tokens }); + } + } +} + +/** + * Add response attributes from LLM result + */ +export function addLLMResponseAttributes(span: Span, response: LangChainLLMResult, recordOutputs: boolean): void { + if (!response) return; + + // Extract finish reasons + if (response.generations && Array.isArray(response.generations)) { + const finishReasons = response.generations + .map(gen => gen.generation_info?.finish_reason) + .filter(reason => typeof reason === 'string'); + + if (finishReasons.length > 0) { + span.setAttributes({ + [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: JSON.stringify(finishReasons), + }); + } + + // Extract response text if recordOutputs is enabled + if (recordOutputs) { + const responseTexts = response.generations + .flat() + .map(gen => gen.text || gen.message?.content) + .filter(text => typeof text === 'string'); + + if (responseTexts.length > 0) { + span.setAttributes({ + [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: JSON.stringify(responseTexts), + }); + } + + // Extract tool calls from message content if present + extractToolCallsAttributes(response.generations as LangChainMessage[][], span); + } + } + + // Extract token usage - handle both formats (OpenAI and Anthropic) + extractTokenUsageAttributes(response.llmOutput, span); + + // Extract model name from response (handle both model_name and model fields) + const modelName = response.llmOutput?.model_name || response.llmOutput?.model; + + if (modelName) { + span.setAttributes({ [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: JSON.stringify(modelName) }); + } + + // Add response ID (useful for debugging/correlation) + if (response.llmOutput?.id) { + span.setAttributes({ 'gen_ai.response.id': JSON.stringify(response.llmOutput.id) }); + } + + // Add stop reason as finish reason if not already captured + if (response.llmOutput?.stop_reason) { + span.setAttributes({ 'gen_ai.response.stop_reason': JSON.stringify(response.llmOutput.stop_reason) }); + } +} diff --git a/packages/google-cloud-serverless/src/index.ts b/packages/google-cloud-serverless/src/index.ts index db52cf357a16..3213abd8ada5 100644 --- a/packages/google-cloud-serverless/src/index.ts +++ b/packages/google-cloud-serverless/src/index.ts @@ -56,6 +56,7 @@ export { onUncaughtExceptionIntegration, onUnhandledRejectionIntegration, openAIIntegration, + langChainIntegration, modulesIntegration, contextLinesIntegration, nodeContextIntegration, diff --git a/packages/node/src/index.ts b/packages/node/src/index.ts index b599351b5124..e469fd75d2d2 100644 --- a/packages/node/src/index.ts +++ b/packages/node/src/index.ts @@ -27,6 +27,7 @@ export { vercelAIIntegration } from './integrations/tracing/vercelai'; export { openAIIntegration } from './integrations/tracing/openai'; export { anthropicAIIntegration } from './integrations/tracing/anthropic-ai'; export { googleGenAIIntegration } from './integrations/tracing/google-genai'; +export { langChainIntegration } from './integrations/tracing/langchain'; export { launchDarklyIntegration, buildLaunchDarklyFlagUsedHandler, @@ -134,6 +135,7 @@ export { consoleIntegration, wrapMcpServerWithSentry, featureFlagsIntegration, + createLangChainCallbackHandler, } from '@sentry/core'; export type { diff --git a/packages/node/src/integrations/tracing/index.ts b/packages/node/src/integrations/tracing/index.ts index dd9d9ac8df2b..2782d7907349 100644 --- a/packages/node/src/integrations/tracing/index.ts +++ b/packages/node/src/integrations/tracing/index.ts @@ -13,6 +13,7 @@ import { hapiIntegration, instrumentHapi } from './hapi'; import { honoIntegration, instrumentHono } from './hono'; import { instrumentKafka, kafkaIntegration } from './kafka'; import { instrumentKoa, koaIntegration } from './koa'; +import { instrumentLangChain, langChainIntegration } from './langchain'; import { instrumentLruMemoizer, lruMemoizerIntegration } from './lrumemoizer'; import { instrumentMongo, mongoIntegration } from './mongo'; import { instrumentMongoose, mongooseIntegration } from './mongoose'; @@ -56,6 +57,7 @@ export function getAutoPerformanceIntegrations(): Integration[] { firebaseIntegration(), anthropicAIIntegration(), googleGenAIIntegration(), + langChainIntegration(), ]; } @@ -93,5 +95,6 @@ export function getOpenTelemetryInstrumentationToPreload(): (((options?: any) => instrumentFirebase, instrumentAnthropicAi, instrumentGoogleGenAI, + instrumentLangChain, ]; } diff --git a/packages/node/src/integrations/tracing/langchain/index.ts b/packages/node/src/integrations/tracing/langchain/index.ts new file mode 100644 index 000000000000..2319487883ab --- /dev/null +++ b/packages/node/src/integrations/tracing/langchain/index.ts @@ -0,0 +1,109 @@ +import type { IntegrationFn, LangChainOptions } from '@sentry/core'; +import { defineIntegration, LANGCHAIN_INTEGRATION_NAME } from '@sentry/core'; +import { generateInstrumentOnce } from '@sentry/node-core'; +import { SentryLangChainInstrumentation } from './instrumentation'; + +export const instrumentLangChain = generateInstrumentOnce( + LANGCHAIN_INTEGRATION_NAME, + options => new SentryLangChainInstrumentation(options), +); + +const _langChainIntegration = ((options: LangChainOptions = {}) => { + return { + name: LANGCHAIN_INTEGRATION_NAME, + setupOnce() { + instrumentLangChain(options); + }, + }; +}) satisfies IntegrationFn; + +/** + * Adds Sentry tracing instrumentation for LangChain. + * + * This integration is enabled by default. + * + * When configured, this integration automatically instruments LangChain runnable instances + * to capture telemetry data by injecting Sentry callback handlers into all LangChain calls. + * + * @example + * ```javascript + * import * as Sentry from '@sentry/node'; + * import { ChatOpenAI } from '@langchain/openai'; + * + * Sentry.init({ + * integrations: [Sentry.langChainIntegration()], + * sendDefaultPii: true, // Enable to record inputs/outputs + * }); + * + * // LangChain calls are automatically instrumented + * const model = new ChatOpenAI(); + * await model.invoke("What is the capital of France?"); + * ``` + * + * ## Manual Callback Handler + * + * You can also manually add the Sentry callback handler alongside other callbacks: + * + * @example + * ```javascript + * import * as Sentry from '@sentry/node'; + * import { createLangChainCallbackHandler } from '@sentry/core'; + * import { ChatOpenAI } from '@langchain/openai'; + * + * const sentryHandler = createLangChainCallbackHandler({ + * recordInputs: true, + * recordOutputs: true + * }); + * + * const model = new ChatOpenAI(); + * await model.invoke( + * "What is the capital of France?", + * { callbacks: [sentryHandler, myOtherCallback] } + * ); + * ``` + * + * ## Options + * + * - `recordInputs`: Whether to record input messages/prompts (default: respects `sendDefaultPii` client option) + * - `recordOutputs`: Whether to record response text (default: respects `sendDefaultPii` client option) + * + * ### Default Behavior + * + * By default, the integration will: + * - Record inputs and outputs ONLY if `sendDefaultPii` is set to `true` in your Sentry client options + * - Otherwise, inputs and outputs are NOT recorded unless explicitly enabled + * + * @example + * ```javascript + * // Record inputs and outputs when sendDefaultPii is false + * Sentry.init({ + * integrations: [ + * Sentry.langChainIntegration({ + * recordInputs: true, + * recordOutputs: true + * }) + * ], + * }); + * + * // Never record inputs/outputs regardless of sendDefaultPii + * Sentry.init({ + * sendDefaultPii: true, + * integrations: [ + * Sentry.langChainIntegration({ + * recordInputs: false, + * recordOutputs: false + * }) + * ], + * }); + * ``` + * + * ## Supported Events + * + * The integration captures the following LangChain lifecycle events: + * - LLM/Chat Model: start, new token (streaming), end, error + * - Chain: start, end, error + * - Tool: start, end, error + * - Retriever: start, end, error + * + */ +export const langChainIntegration = defineIntegration(_langChainIntegration); diff --git a/packages/node/src/integrations/tracing/langchain/instrumentation.ts b/packages/node/src/integrations/tracing/langchain/instrumentation.ts new file mode 100644 index 000000000000..f6a015019f26 --- /dev/null +++ b/packages/node/src/integrations/tracing/langchain/instrumentation.ts @@ -0,0 +1,201 @@ +import { + type InstrumentationConfig, + type InstrumentationModuleDefinition, + InstrumentationBase, + InstrumentationNodeModuleDefinition, + InstrumentationNodeModuleFile, +} from '@opentelemetry/instrumentation'; +import type { LangChainOptions } from '@sentry/core'; +import { createLangChainCallbackHandler, getClient, SDK_VERSION } from '@sentry/core'; + +const supportedVersions = ['>=0.1.0 <1.0.0']; + +type LangChainInstrumentationOptions = InstrumentationConfig & LangChainOptions; + +/** + * Represents the patched shape of LangChain provider package exports + */ +interface PatchedLangChainExports { + [key: string]: unknown; +} + +/** + * Augments a callback handler list with Sentry's handler if not already present + */ +function augmentCallbackHandlers(handlers: unknown, sentryHandler: unknown): unknown { + // Handle null/undefined - return array with just our handler + if (!handlers) { + return [sentryHandler]; + } + + // If handlers is already an array + if (Array.isArray(handlers)) { + // Check if our handler is already in the list + if (handlers.includes(sentryHandler)) { + return handlers; + } + // Add our handler to the list + return [...handlers, sentryHandler]; + } + + // If it's a single handler object, convert to array + if (typeof handlers === 'object') { + return [handlers, sentryHandler]; + } + + // Unknown type - return original + return handlers; +} + +/** + * Wraps Runnable methods (invoke, stream, batch) to inject Sentry callbacks at request time + * Uses a Proxy to intercept method calls and augment the options.callbacks + */ +function wrapRunnableMethod( + originalMethod: (...args: unknown[]) => unknown, + sentryHandler: unknown, + _methodName: string, +): (...args: unknown[]) => unknown { + return new Proxy(originalMethod, { + apply(target, thisArg, args: unknown[]): unknown { + // LangChain Runnable method signatures: + // invoke(input, options?) - options contains callbacks + // stream(input, options?) - options contains callbacks + // batch(inputs, options?) - options contains callbacks + + // Options is typically the second argument + const optionsIndex = 1; + let options = args[optionsIndex] as Record | undefined; + + // If options don't exist or aren't an object, create them + if (!options || typeof options !== 'object' || Array.isArray(options)) { + options = {}; + args[optionsIndex] = options; + } + + // Inject our callback handler into options.callbacks (request time callbacks) + const existingCallbacks = options.callbacks; + const augmentedCallbacks = augmentCallbackHandlers(existingCallbacks, sentryHandler); + options.callbacks = augmentedCallbacks; + + // Call original method with augmented options + return Reflect.apply(target, thisArg, args); + }, + }) as (...args: unknown[]) => unknown; +} + +/** + * Sentry LangChain instrumentation using OpenTelemetry. + */ +export class SentryLangChainInstrumentation extends InstrumentationBase { + public constructor(config: LangChainInstrumentationOptions = {}) { + super('@sentry/instrumentation-langchain', SDK_VERSION, config); + } + + /** + * Initializes the instrumentation by defining the modules to be patched. + * We patch the BaseChatModel class methods to inject callbacks + * + * We hook into provider packages (@langchain/anthropic, @langchain/openai, etc.) + * because @langchain/core is often bundled and not loaded as a separate module + */ + public init(): InstrumentationModuleDefinition | InstrumentationModuleDefinition[] { + const modules: InstrumentationModuleDefinition[] = []; + + // Hook into common LangChain provider packages + const providerPackages = [ + '@langchain/anthropic', + '@langchain/openai', + '@langchain/google-genai', + '@langchain/mistralai', + '@langchain/google-vertexai ', + '@langchain/groq', + ]; + + for (const packageName of providerPackages) { + // In CJS, LangChain packages re-export from dist/index.cjs files. + // Patching only the root module sometimes misses the real implementation or + // gets overwritten when that file is loaded. We add a file-level patch so that + // _patch runs again on the concrete implementation + modules.push( + new InstrumentationNodeModuleDefinition( + packageName, + supportedVersions, + this._patch.bind(this), + exports => exports, + [ + new InstrumentationNodeModuleFile( + `${packageName}/dist/index.cjs`, + supportedVersions, + this._patch.bind(this), + exports => exports, + ), + ], + ), + ); + } + + return modules; + } + + /** + * Core patch logic - patches chat model methods to inject Sentry callbacks + * This is called when a LangChain provider package is loaded + */ + private _patch(exports: PatchedLangChainExports): PatchedLangChainExports | void { + const client = getClient(); + const defaultPii = Boolean(client?.getOptions().sendDefaultPii); + + const config = this.getConfig(); + + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + const recordInputs = config?.recordInputs ?? defaultPii; + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + const recordOutputs = config?.recordOutputs ?? defaultPii; + + // Create a shared handler instance + const sentryHandler = createLangChainCallbackHandler({ + recordInputs, + recordOutputs, + }); + + // Patch Runnable methods to inject callbacks at request time + // This directly manipulates options.callbacks that LangChain uses + this._patchRunnableMethods(exports, sentryHandler); + + return exports; + } + + /** + * Patches chat model methods (invoke, stream, batch) to inject Sentry callbacks + * Finds a chat model class from the provider package exports and patches its prototype methods + */ + private _patchRunnableMethods(exports: PatchedLangChainExports, sentryHandler: unknown): void { + // Find a chat model class in the exports (e.g., ChatAnthropic, ChatOpenAI) + const chatModelClass = Object.values(exports).find( + exp => typeof exp === 'function' && exp.name?.includes('Chat'), + ) as { prototype: unknown; name: string } | undefined; + + if (!chatModelClass) { + return; + } + + // Patch directly on chatModelClass.prototype + const targetProto = chatModelClass.prototype as Record; + + // Patch the methods (invoke, stream, batch) + // All chat model instances will inherit these patched methods + const methodsToPatch = ['invoke', 'stream', 'batch'] as const; + + for (const methodName of methodsToPatch) { + const method = targetProto[methodName]; + if (typeof method === 'function') { + targetProto[methodName] = wrapRunnableMethod( + method as (...args: unknown[]) => unknown, + sentryHandler, + methodName, + ); + } + } + } +} diff --git a/yarn.lock b/yarn.lock index c0bc7ba27923..0aee43c39414 100644 --- a/yarn.lock +++ b/yarn.lock @@ -335,6 +335,13 @@ dependencies: json-schema-to-ts "^3.1.1" +"@anthropic-ai/sdk@^0.65.0": + version "0.65.0" + resolved "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.65.0.tgz#3f464fe2029eacf8e7e7fb8197579d00c8ca7502" + integrity sha512-zIdPOcrCVEI8t3Di40nH4z9EoeyGZfXbYSvWdDLsB/KkaSYMnEgC7gmcgWu83g2NTn1ZTpbMvpdttWDGGIk6zw== + dependencies: + json-schema-to-ts "^3.1.1" + "@apm-js-collab/code-transformer@^0.8.0", "@apm-js-collab/code-transformer@^0.8.2": version "0.8.2" resolved "https://registry.yarnpkg.com/@apm-js-collab/code-transformer/-/code-transformer-0.8.2.tgz#a3160f16d1c4df9cb81303527287ad18d00994d1" @@ -2678,6 +2685,11 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz#bbe12dca5b4ef983a0d0af4b07b9bc90ea0ababa" integrity sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA== +"@cfworker/json-schema@^4.0.2": + version "4.1.1" + resolved "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz#4a2a3947ee9fa7b7c24be981422831b8674c3be6" + integrity sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og== + "@cloudflare/kv-asset-handler@0.4.0", "@cloudflare/kv-asset-handler@^0.4.0": version "0.4.0" resolved "https://registry.yarnpkg.com/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.4.0.tgz#a8588c6a2e89bb3e87fb449295a901c9f6d3e1bf" @@ -4896,6 +4908,32 @@ resolved "https://registry.yarnpkg.com/@kwsites/promise-deferred/-/promise-deferred-1.1.1.tgz#8ace5259254426ccef57f3175bc64ed7095ed919" integrity sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw== +"@langchain/anthropic@^0.3.10": + version "0.3.31" + resolved "https://registry.npmjs.org/@langchain/anthropic/-/anthropic-0.3.31.tgz#80bc2464ab98cfb8df0de50cf219d92cfe5934e1" + integrity sha512-XyjwE1mA1I6sirSlVZtI6tyv7nH3+b8F5IFDi9WNKA8+SidJ0o3cP90TxrK7x1sSLmdj+su3f8s2hOusw6xpaw== + dependencies: + "@anthropic-ai/sdk" "^0.65.0" + fast-xml-parser "^4.4.1" + +"@langchain/core@^0.3.28": + version "0.3.78" + resolved "https://registry.npmjs.org/@langchain/core/-/core-0.3.78.tgz#40e69fba6688858edbcab4473358ec7affc685fd" + integrity sha512-Nn0x9erQlK3zgtRU1Z8NUjLuyW0gzdclMsvLQ6wwLeDqV91pE+YKl6uQb+L2NUDs4F0N7c2Zncgz46HxrvPzuA== + dependencies: + "@cfworker/json-schema" "^4.0.2" + ansi-styles "^5.0.0" + camelcase "6" + decamelize "1.2.0" + js-tiktoken "^1.0.12" + langsmith "^0.3.67" + mustache "^4.2.0" + p-queue "^6.6.2" + p-retry "4" + uuid "^10.0.0" + zod "^3.25.32" + zod-to-json-schema "^3.22.3" + "@leichtgewicht/ip-codec@^2.0.1": version "2.0.4" resolved "https://registry.yarnpkg.com/@leichtgewicht/ip-codec/-/ip-codec-2.0.4.tgz#b2ac626d6cb9c8718ab459166d4bb405b8ffa78b" @@ -9018,6 +9056,11 @@ resolved "https://registry.yarnpkg.com/@types/unist/-/unist-2.0.8.tgz#bb197b9639aa1a04cf464a617fe800cccd92ad5c" integrity sha512-d0XxK3YTObnWVp6rZuev3c49+j4Lo8g4L1ZRm9z5L0xpoZycUPshHgczK5gsUMaZOstjVYYi09p5gYvUtfChYw== +"@types/uuid@^10.0.0": + version "10.0.0" + resolved "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz#e9c07fe50da0f53dc24970cca94d619ff03f6f6d" + integrity sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ== + "@types/webidl-conversions@*": version "7.0.3" resolved "https://registry.yarnpkg.com/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz#1306dbfa53768bcbcfc95a1c8cde367975581859" @@ -11426,7 +11469,7 @@ base64-arraybuffer@^1.0.1: resolved "https://registry.yarnpkg.com/base64-arraybuffer/-/base64-arraybuffer-1.0.2.tgz#1c37589a7c4b0746e34bd1feb951da2df01c1bdc" integrity sha512-I3yl4r9QB5ZRY3XuJVEPfc2XhZO6YweFPI+UovAzn+8/hb3oJ6lnysaFcjVpkCPfVWFUDvoZ8kmVDP7WyRtYtQ== -base64-js@^1.2.0, base64-js@^1.3.0, base64-js@^1.3.1: +base64-js@^1.2.0, base64-js@^1.3.0, base64-js@^1.3.1, base64-js@^1.5.1: version "1.5.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== @@ -12500,6 +12543,11 @@ camelcase@5.0.0: resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.0.0.tgz#03295527d58bd3cd4aa75363f35b2e8d97be2f42" integrity sha512-faqwZqnWxbxn+F1d399ygeamQNy3lPp/H9H6rNrqYh4FSVCtcY+3cub1MxA8o9mDd55mM8Aghuu/kuyYA6VTsA== +camelcase@6, camelcase@^6.2.0, camelcase@^6.3.0: + version "6.3.0" + resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a" + integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== + camelcase@^5.3.1: version "5.3.1" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320" @@ -12509,7 +12557,6 @@ camelcase@^6.3.0: version "6.3.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a" integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== - camelcase@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-7.0.1.tgz#f02e50af9fd7782bc8b88a3558c32fd3a388f048" @@ -13232,6 +13279,13 @@ console-control-strings@^1.1.0: resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e" integrity sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4= +console-table-printer@^2.12.1: + version "2.14.6" + resolved "https://registry.npmjs.org/console-table-printer/-/console-table-printer-2.14.6.tgz#edfe0bf311fa2701922ed509443145ab51e06436" + integrity sha512-MCBl5HNVaFuuHW6FGbL/4fB7N/ormCy+tQ+sxTrF6QtSbSNETvPuOVbkJBhzDgYhvjWGrTma4eYJa37ZuoQsPw== + dependencies: + simple-wcswidth "^1.0.1" + console-ui@^3.0.4, console-ui@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/console-ui/-/console-ui-3.1.2.tgz#51aef616ff02013c85ccee6a6d77ef7a94202e7a" @@ -13948,7 +14002,7 @@ decamelize-keys@^1.1.0: decamelize "^1.1.0" map-obj "^1.0.0" -decamelize@^1.1.0: +decamelize@1.2.0, decamelize@^1.1.0: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" integrity sha1-9lNNFRSCabIDUue+4m9QH5oZEpA= @@ -20079,6 +20133,13 @@ js-string-escape@^1.0.1: resolved "https://registry.yarnpkg.com/js-string-escape/-/js-string-escape-1.0.1.tgz#e2625badbc0d67c7533e9edc1068c587ae4137ef" integrity sha1-4mJbrbwNZ8dTPp7cEGjFh65BN+8= +js-tiktoken@^1.0.12: + version "1.0.21" + resolved "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz#368a9957591a30a62997dd0c4cf30866f00f8221" + integrity sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g== + dependencies: + base64-js "^1.5.1" + "js-tokens@^3.0.0 || ^4.0.0", js-tokens@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" @@ -20483,6 +20544,19 @@ lambda-local@^2.2.0: dotenv "^16.3.1" winston "^3.10.0" +langsmith@^0.3.67: + version "0.3.74" + resolved "https://registry.npmjs.org/langsmith/-/langsmith-0.3.74.tgz#014d31a9ff7530b54f0d797502abd512ce8fb6fb" + integrity sha512-ZuW3Qawz8w88XcuCRH91yTp6lsdGuwzRqZ5J0Hf5q/AjMz7DwcSv0MkE6V5W+8hFMI850QZN2Wlxwm3R9lHlZg== + dependencies: + "@types/uuid" "^10.0.0" + chalk "^4.1.2" + console-table-printer "^2.12.1" + p-queue "^6.6.2" + p-retry "4" + semver "^7.6.3" + uuid "^10.0.0" + language-subtag-registry@~0.3.2: version "0.3.22" resolved "https://registry.yarnpkg.com/language-subtag-registry/-/language-subtag-registry-0.3.22.tgz#2e1500861b2e457eba7e7ae86877cbd08fa1fd1d" @@ -24173,7 +24247,7 @@ p-pipe@3.1.0: resolved "https://registry.npmjs.org/p-pipe/-/p-pipe-3.1.0.tgz#48b57c922aa2e1af6a6404cb7c6bf0eb9cc8e60e" integrity sha512-08pj8ATpzMR0Y80x50yJHn37NF6vjrqHutASaX5LiH5npS9XPvrUmscd9MF5R4fuYRHOxQR1FfMIlF7AzwoPqw== -p-queue@6.6.2: +p-queue@6.6.2, p-queue@^6.6.2: version "6.6.2" resolved "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz#2068a9dcf8e67dd0ec3e7a2bcb76810faa85e426" integrity sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ== @@ -24194,7 +24268,7 @@ p-reduce@2.1.0, p-reduce@^2.0.0, p-reduce@^2.1.0: resolved "https://registry.yarnpkg.com/p-reduce/-/p-reduce-2.1.0.tgz#09408da49507c6c274faa31f28df334bc712b64a" integrity sha512-2USApvnsutq8uoxZBGbbWM0JIYLiEMJ9RlaN7fAzVNb9OZN0SHjjTTfIcb667XynS5Y1VhwDJVDa72TnPzAYWw== -p-retry@^4.5.0: +p-retry@4, p-retry@^4.5.0: version "4.6.2" resolved "https://registry.yarnpkg.com/p-retry/-/p-retry-4.6.2.tgz#9baae7184057edd4e17231cee04264106e092a16" integrity sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ== @@ -27844,6 +27918,11 @@ simple-update-notifier@^2.0.0: dependencies: semver "^7.5.3" +simple-wcswidth@^1.0.1: + version "1.1.2" + resolved "https://registry.npmjs.org/simple-wcswidth/-/simple-wcswidth-1.1.2.tgz#66722f37629d5203f9b47c5477b1225b85d6525b" + integrity sha512-j7piyCjAeTDSjzTSQ7DokZtMNwNlEAyxqSZeCS+CXH7fJ4jx3FuJ/mTW3mE+6JLs4VJBbcll0Kjn+KXI5t21Iw== + sinon@19.0.2: version "19.0.2" resolved "https://registry.yarnpkg.com/sinon/-/sinon-19.0.2.tgz#944cf771d22236aa84fc1ab70ce5bffc3a215dad" @@ -30555,6 +30634,11 @@ uuid@8.3.2, uuid@^8.0.0, uuid@^8.3.0, uuid@^8.3.2: resolved "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2" integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== +uuid@^10.0.0: + version "10.0.0" + resolved "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz#5a95aa454e6e002725c79055fd42aaba30ca6294" + integrity sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ== + uuid@^11.1.0: version "11.1.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-11.1.0.tgz#9549028be1753bb934fc96e2bca09bb4105ae912" @@ -31970,6 +32054,11 @@ zip-stream@^6.0.1: compress-commons "^6.0.2" readable-stream "^4.0.0" +zod-to-json-schema@^3.22.3: + version "3.24.6" + resolved "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.6.tgz#5920f020c4d2647edfbb954fa036082b92c9e12d" + integrity sha512-h/z3PKvcTcTetyjl1fkj79MHNEjm+HpD6NXheWjzOekY7kV+lwDYnHw+ivHkijnCSMz1yJaWBD9vu/Fcmk+vEg== + zod-to-json-schema@^3.24.1: version "3.24.5" resolved "https://registry.yarnpkg.com/zod-to-json-schema/-/zod-to-json-schema-3.24.5.tgz#d1095440b147fb7c2093812a53c54df8d5df50a3" @@ -31985,6 +32074,11 @@ zod@^3.22.2, zod@^3.22.4, zod@^3.23.8, zod@^3.24.1: resolved "https://registry.yarnpkg.com/zod/-/zod-3.25.75.tgz#8ff9be2fbbcb381a9236f9f74a8879ca29dcc504" integrity sha512-OhpzAmVzabPOL6C3A3gpAifqr9MqihV/Msx3gor2b2kviCgcb+HM9SEOpMWwwNp9MRunWnhtAKUoo0AHhjyPPg== +zod@^3.25.32: + version "3.25.76" + resolved "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz#26841c3f6fd22a6a2760e7ccb719179768471e34" + integrity sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ== + zone.js@^0.12.0: version "0.12.0" resolved "https://registry.yarnpkg.com/zone.js/-/zone.js-0.12.0.tgz#a4a6e5fab6d34bd37d89c77e89ac2e6f4a3d2c30" From c8d9d4b56b01e20ac4b91a23f027e6f2b7b331db Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Fri, 17 Oct 2025 11:28:13 +0200 Subject: [PATCH 02/12] quick refactors --- .../tracing/langchain/instrument-with-pii.mjs | 2 + .../suites/tracing/langchain/instrument.mjs | 2 + .../suites/tracing/langchain/test.ts | 36 +- packages/core/src/index.ts | 10 +- .../core/src/utils/ai/gen-ai-attributes.ts | 15 + .../core/src/utils/langchain/constants.ts | 29 +- packages/core/src/utils/langchain/index.ts | 81 ++-- packages/core/src/utils/langchain/types.ts | 34 -- packages/core/src/utils/langchain/utils.ts | 439 ++++++++++-------- .../integrations/tracing/langchain/index.ts | 3 +- .../tracing/langchain/instrumentation.ts | 21 +- 11 files changed, 336 insertions(+), 336 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-with-pii.mjs index cb68a6f7683e..85b2a963d977 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-with-pii.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/instrument-with-pii.mjs @@ -7,6 +7,8 @@ Sentry.init({ tracesSampleRate: 1.0, sendDefaultPii: true, transport: loggingTransport, + // Filter out Anthropic integration to avoid duplicate spans with LangChain + integrations: integrations => integrations.filter(integration => integration.name !== 'Anthropic_AI'), beforeSendTransaction: event => { // Filter out mock express server transactions if (event.transaction.includes('/v1/messages')) { diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/instrument.mjs index b4ce44f3e91a..524d19f4b995 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/instrument.mjs @@ -7,6 +7,8 @@ Sentry.init({ tracesSampleRate: 1.0, sendDefaultPii: false, transport: loggingTransport, + // Filter out Anthropic integration to avoid duplicate spans with LangChain + integrations: integrations => integrations.filter(integration => integration.name !== 'Anthropic_AI'), beforeSendTransaction: event => { // Filter out mock express server transactions if (event.transaction.includes('/v1/messages')) { diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts index 7bd9c416e5fa..357b50e78488 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts @@ -15,8 +15,8 @@ describe('LangChain integration', () => { 'gen_ai.operation.name': 'chat', 'sentry.op': 'gen_ai.chat', 'sentry.origin': 'auto.ai.langchain', - 'gen_ai.system': '"anthropic"', - 'gen_ai.request.model': '"claude-3-5-sonnet-20241022"', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-3-5-sonnet-20241022', 'gen_ai.request.temperature': 0.7, 'gen_ai.request.max_tokens': 100, 'gen_ai.usage.input_tokens': 10, @@ -26,7 +26,7 @@ describe('LangChain integration', () => { 'gen_ai.response.model': expect.any(String), 'gen_ai.response.stop_reason': expect.any(String), }), - description: 'chat "claude-3-5-sonnet-20241022"', + description: 'chat claude-3-5-sonnet-20241022', op: 'gen_ai.chat', origin: 'auto.ai.langchain', status: 'ok', @@ -37,8 +37,8 @@ describe('LangChain integration', () => { 'gen_ai.operation.name': 'chat', 'sentry.op': 'gen_ai.chat', 'sentry.origin': 'auto.ai.langchain', - 'gen_ai.system': '"anthropic"', - 'gen_ai.request.model': '"claude-3-opus-20240229"', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-3-opus-20240229', 'gen_ai.request.temperature': 0.9, 'gen_ai.request.top_p': 0.95, 'gen_ai.request.max_tokens': 200, @@ -49,7 +49,7 @@ describe('LangChain integration', () => { 'gen_ai.response.model': expect.any(String), 'gen_ai.response.stop_reason': expect.any(String), }), - description: 'chat "claude-3-opus-20240229"', + description: 'chat claude-3-opus-20240229', op: 'gen_ai.chat', origin: 'auto.ai.langchain', status: 'ok', @@ -60,10 +60,10 @@ describe('LangChain integration', () => { 'gen_ai.operation.name': 'chat', 'sentry.op': 'gen_ai.chat', 'sentry.origin': 'auto.ai.langchain', - 'gen_ai.system': '"anthropic"', - 'gen_ai.request.model': '"error-model"', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'error-model', }), - description: 'chat "error-model"', + description: 'chat error-model', op: 'gen_ai.chat', origin: 'auto.ai.langchain', status: 'unknown_error', @@ -80,8 +80,8 @@ describe('LangChain integration', () => { 'gen_ai.operation.name': 'chat', 'sentry.op': 'gen_ai.chat', 'sentry.origin': 'auto.ai.langchain', - 'gen_ai.system': '"anthropic"', - 'gen_ai.request.model': '"claude-3-5-sonnet-20241022"', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-3-5-sonnet-20241022', 'gen_ai.request.temperature': 0.7, 'gen_ai.request.max_tokens': 100, 'gen_ai.request.messages': expect.any(String), // Should include messages when recordInputs: true @@ -93,7 +93,7 @@ describe('LangChain integration', () => { 'gen_ai.usage.output_tokens': 15, 'gen_ai.usage.total_tokens': 25, }), - description: 'chat "claude-3-5-sonnet-20241022"', + description: 'chat claude-3-5-sonnet-20241022', op: 'gen_ai.chat', origin: 'auto.ai.langchain', status: 'ok', @@ -104,8 +104,8 @@ describe('LangChain integration', () => { 'gen_ai.operation.name': 'chat', 'sentry.op': 'gen_ai.chat', 'sentry.origin': 'auto.ai.langchain', - 'gen_ai.system': '"anthropic"', - 'gen_ai.request.model': '"claude-3-opus-20240229"', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-3-opus-20240229', 'gen_ai.request.temperature': 0.9, 'gen_ai.request.top_p': 0.95, 'gen_ai.request.max_tokens': 200, @@ -118,7 +118,7 @@ describe('LangChain integration', () => { 'gen_ai.usage.output_tokens': 15, 'gen_ai.usage.total_tokens': 25, }), - description: 'chat "claude-3-opus-20240229"', + description: 'chat claude-3-opus-20240229', op: 'gen_ai.chat', origin: 'auto.ai.langchain', status: 'ok', @@ -129,11 +129,11 @@ describe('LangChain integration', () => { 'gen_ai.operation.name': 'chat', 'sentry.op': 'gen_ai.chat', 'sentry.origin': 'auto.ai.langchain', - 'gen_ai.system': '"anthropic"', - 'gen_ai.request.model': '"error-model"', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'error-model', 'gen_ai.request.messages': expect.any(String), // Should include messages when recordInputs: true }), - description: 'chat "error-model"', + description: 'chat error-model', op: 'gen_ai.chat', origin: 'auto.ai.langchain', status: 'unknown_error', diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 2d2ea817cd92..38ed0bab7f09 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -159,15 +159,7 @@ export type { GoogleGenAIOptions, GoogleGenAIIstrumentedMethod, } from './utils/google-genai/types'; -export type { - LangChainOptions, - LangChainIntegration, - LangChainSerializedLLM, - LangChainMessage, - LangChainLLMResult, - LangChainTool, - LangChainDocument, -} from './utils/langchain/types'; +export type { LangChainOptions, LangChainIntegration } from './utils/langchain/types'; export type { FeatureFlag } from './utils/featureFlags'; export { diff --git a/packages/core/src/utils/ai/gen-ai-attributes.ts b/packages/core/src/utils/ai/gen-ai-attributes.ts index 9124602644e4..00adcb202613 100644 --- a/packages/core/src/utils/ai/gen-ai-attributes.ts +++ b/packages/core/src/utils/ai/gen-ai-attributes.ts @@ -80,6 +80,11 @@ export const GEN_AI_RESPONSE_MODEL_ATTRIBUTE = 'gen_ai.response.model'; */ export const GEN_AI_RESPONSE_ID_ATTRIBUTE = 'gen_ai.response.id'; +/** + * The reason why the model stopped generating tokens + */ +export const GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE = 'gen_ai.response.stop_reason'; + /** * The number of tokens used in the prompt */ @@ -129,6 +134,16 @@ export const GEN_AI_RESPONSE_STREAMING_ATTRIBUTE = 'gen_ai.response.streaming'; */ export const GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE = 'gen_ai.response.tool_calls'; +/** + * The number of cache creation input tokens used + */ +export const GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS_ATTRIBUTE = 'gen_ai.usage.cache_creation_input_tokens'; + +/** + * The number of cache read input tokens used + */ +export const GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_ATTRIBUTE = 'gen_ai.usage.cache_read_input_tokens'; + // ============================================================================= // OPENAI-SPECIFIC ATTRIBUTES // ============================================================================= diff --git a/packages/core/src/utils/langchain/constants.ts b/packages/core/src/utils/langchain/constants.ts index f15476b41ae4..ead9bed623ad 100644 --- a/packages/core/src/utils/langchain/constants.ts +++ b/packages/core/src/utils/langchain/constants.ts @@ -1,24 +1,11 @@ export const LANGCHAIN_INTEGRATION_NAME = 'LangChain'; export const LANGCHAIN_ORIGIN = 'auto.ai.langchain'; -/** - * LangChain event types we instrument - * Based on LangChain.js callback system - * @see https://js.langchain.com/docs/concepts/callbacks/ - */ -export const LANGCHAIN_EVENT_TYPES = { - CHAT_MODEL_START: 'handleChatModelStart', - LLM_START: 'handleLLMStart', - LLM_NEW_TOKEN: 'handleLLMNewToken', - LLM_END: 'handleLLMEnd', - LLM_ERROR: 'handleLLMError', - CHAIN_START: 'handleChainStart', - CHAIN_END: 'handleChainEnd', - CHAIN_ERROR: 'handleChainError', - TOOL_START: 'handleToolStart', - TOOL_END: 'handleToolEnd', - TOOL_ERROR: 'handleToolError', - RETRIEVER_START: 'handleRetrieverStart', - RETRIEVER_END: 'handleRetrieverEnd', - RETRIEVER_ERROR: 'handleRetrieverError', -} as const; +export const ROLE_MAP: Record = { + human: 'user', + ai: 'assistant', + assistant: 'assistant', + system: 'system', + function: 'function', + tool: 'tool', +}; diff --git a/packages/core/src/utils/langchain/index.ts b/packages/core/src/utils/langchain/index.ts index 1130bd65ae8e..81e3f8eaf072 100644 --- a/packages/core/src/utils/langchain/index.ts +++ b/packages/core/src/utils/langchain/index.ts @@ -5,17 +5,11 @@ import { startSpanManual } from '../../tracing/trace'; import type { Span, SpanAttributeValue } from '../../types-hoist/span'; import { GEN_AI_OPERATION_NAME_ATTRIBUTE, GEN_AI_REQUEST_MODEL_ATTRIBUTE } from '../ai/gen-ai-attributes'; import { LANGCHAIN_ORIGIN } from './constants'; -import type { - LangChainLLMResult, - LangChainMessage, - LangChainOptions, - LangChainRunId, - LangChainSerializedLLM, -} from './types'; +import type { LangChainLLMResult, LangChainMessage, LangChainOptions, LangChainSerializedLLM } from './types'; import { - addLLMResponseAttributes, extractChatModelRequestAttributes, extractLLMRequestAttributes, + extractLlmResponseAttributes, getInvocationParams, } from './utils'; @@ -29,8 +23,8 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): handleLLMStart?: ( llm: LangChainSerializedLLM, prompts: string[], - runId: LangChainRunId, - parentRunId?: LangChainRunId, + runId: string, + parentRunId?: string, extraParams?: Record, tags?: string[] | Record, metadata?: Record, @@ -39,43 +33,38 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): handleChatModelStart?: ( llm: LangChainSerializedLLM, messages: LangChainMessage[][], - runId: LangChainRunId, - parentRunId?: LangChainRunId, + runId: string, + parentRunId?: string, extraParams?: Record, tags?: string[] | Record, metadata?: Record, runName?: string | Record, ) => void; - handleLLMNewToken?: (token: string, runId: LangChainRunId) => void; - handleLLMEnd?: (output: LangChainLLMResult, runId: LangChainRunId) => void; - handleLLMError?: (error: Error, runId: LangChainRunId) => void; + handleLLMNewToken?: (token: string, runId: string) => void; + handleLLMEnd?: (output: LangChainLLMResult, runId: string) => void; + handleLLMError?: (error: Error, runId: string) => void; handleChainStart?: ( chain: { name?: string }, inputs: Record, - runId: LangChainRunId, - parentRunId?: LangChainRunId, + runId: string, + parentRunId?: string, ) => void; - handleChainEnd?: (outputs: Record, runId: LangChainRunId) => void; - handleChainError?: (error: Error, runId: LangChainRunId) => void; - handleToolStart?: ( - tool: { name?: string }, - input: string, - runId: LangChainRunId, - parentRunId?: LangChainRunId, - ) => void; - handleToolEnd?: (output: string, runId: LangChainRunId) => void; - handleToolError?: (error: Error, runId: LangChainRunId) => void; + handleChainEnd?: (outputs: Record, runId: string) => void; + handleChainError?: (error: Error, runId: string) => void; + handleToolStart?: (tool: { name?: string }, input: string, runId: string, parentRunId?: string) => void; + handleToolEnd?: (output: string, runId: string) => void; + handleToolError?: (error: Error, runId: string) => void; } { const recordInputs = options.recordInputs ?? false; const recordOutputs = options.recordOutputs ?? false; // Internal state - single instance tracks all spans - const spanMap = new Map(); + const spanMap = new Map(); /** * Exit a span and clean up */ - const exitSpan = (runId: LangChainRunId): void => { + const exitSpan = (runId: string): void => { const span = spanMap.get(runId); if (span) { span.end(); @@ -91,8 +80,8 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): handleLLMStart( llm: LangChainSerializedLLM, prompts: string[], - runId: LangChainRunId, - _parentRunId?: LangChainRunId, + runId: string, + _parentRunId?: string, _extraParams?: Record, tags?: string[] | Record, metadata?: Record, @@ -124,8 +113,8 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): handleChatModelStart( llm: LangChainSerializedLLM, messages: LangChainMessage[][], - runId: LangChainRunId, - _parentRunId?: LangChainRunId, + runId: string, + _parentRunId?: string, _extraParams?: Record, tags?: string[] | Record, metadata?: Record, @@ -157,7 +146,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): // LLM End Handler - note: handleLLMEnd with capital LLM (used by both LLMs and chat models!) handleLLMEnd( output: LangChainLLMResult, - runId: LangChainRunId, + runId: string, _parentRunId?: string, _tags?: string[], _extraParams?: Record, @@ -165,7 +154,10 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): try { const span = spanMap.get(runId); if (span) { - addLLMResponseAttributes(span, output, recordOutputs); + const attributes = extractLlmResponseAttributes(output, recordOutputs); + if (attributes) { + span.setAttributes(attributes); + } exitSpan(runId); } } catch { @@ -174,7 +166,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): }, // LLM Error Handler - note: handleLLMError with capital LLM - handleLLMError(error: Error, runId: LangChainRunId) { + handleLLMError(error: Error, runId: string) { try { const span = spanMap.get(runId); if (span) { @@ -195,12 +187,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): }, // Chain Start Handler - handleChainStart( - chain: { name?: string }, - inputs: Record, - runId: LangChainRunId, - _parentRunId?: LangChainRunId, - ) { + handleChainStart(chain: { name?: string }, inputs: Record, runId: string, _parentRunId?: string) { try { const chainName = chain.name || 'unknown_chain'; const attributes: Record = { @@ -231,7 +218,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): }, // Chain End Handler - handleChainEnd(outputs: Record, runId: LangChainRunId) { + handleChainEnd(outputs: Record, runId: string) { try { const span = spanMap.get(runId); if (span) { @@ -249,7 +236,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): }, // Chain Error Handler - handleChainError(error: Error, runId: LangChainRunId) { + handleChainError(error: Error, runId: string) { try { const span = spanMap.get(runId); if (span) { @@ -269,7 +256,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): }, // Tool Start Handler - handleToolStart(tool: { name?: string }, input: string, runId: LangChainRunId, _parentRunId?: LangChainRunId) { + handleToolStart(tool: { name?: string }, input: string, runId: string, _parentRunId?: string) { try { const toolName = tool.name || 'unknown_tool'; const attributes: Record = { @@ -299,7 +286,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): }, // Tool End Handler - handleToolEnd(output: string, runId: LangChainRunId) { + handleToolEnd(output: string, runId: string) { try { const span = spanMap.get(runId); if (span) { @@ -317,7 +304,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): }, // Tool Error Handler - handleToolError(error: Error, runId: LangChainRunId) { + handleToolError(error: Error, runId: string) { try { const span = spanMap.get(runId); if (span) { diff --git a/packages/core/src/utils/langchain/types.ts b/packages/core/src/utils/langchain/types.ts index 4e18b236d630..62bef5832ab7 100644 --- a/packages/core/src/utils/langchain/types.ts +++ b/packages/core/src/utils/langchain/types.ts @@ -36,7 +36,6 @@ export interface LangChainSerializedLLM { */ export interface LangChainMessage { [key: string]: unknown; - // Regular message format type?: string; content?: string; message?: { @@ -82,39 +81,6 @@ export interface LangChainLLMResult { }; } -/** - * LangChain Run ID - */ -export type LangChainRunId = string; - -/** - * LangChain span context stored per run - */ -export interface LangChainSpanContext { - spanId: string; - traceId?: string; - parentSpanId?: string; - streamingBuffer?: string[]; -} - -/** - * LangChain Tool structure - */ -export interface LangChainTool { - [key: string]: unknown; - name: string; - description?: string; -} - -/** - * LangChain Document structure for retrievers - */ -export interface LangChainDocument { - [key: string]: unknown; - pageContent: string; - metadata?: Record; -} - /** * Integration interface for type safety */ diff --git a/packages/core/src/utils/langchain/utils.ts b/packages/core/src/utils/langchain/utils.ts index 0f9fb317c836..9c0024e632eb 100644 --- a/packages/core/src/utils/langchain/utils.ts +++ b/packages/core/src/utils/langchain/utils.ts @@ -1,4 +1,3 @@ -import type { Span } from '../..'; import { type SpanAttributeValue, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '../..'; import { GEN_AI_OPERATION_NAME_ATTRIBUTE, @@ -11,177 +10,232 @@ import { GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE, GEN_AI_REQUEST_TOP_P_ATTRIBUTE, GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE, + GEN_AI_RESPONSE_ID_ATTRIBUTE, GEN_AI_RESPONSE_MODEL_ATTRIBUTE, + GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE, GEN_AI_RESPONSE_TEXT_ATTRIBUTE, GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE, GEN_AI_SYSTEM_ATTRIBUTE, + GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS_ATTRIBUTE, + GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_ATTRIBUTE, GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE, GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, } from '../ai/gen-ai-attributes'; -import { LANGCHAIN_ORIGIN } from './constants'; +import { LANGCHAIN_ORIGIN, ROLE_MAP } from './constants'; import type { LangChainLLMResult, LangChainMessage, LangChainSerializedLLM } from './types'; /** - * Extract invocation params from tags object - * LangChain passes runtime parameters in the tags object + * Assigns an attribute only when the value is neither `undefined` nor `null`. + * + * We keep this tiny helper because call sites are repetitive and easy to miswrite. + * It also preserves falsy-but-valid values like `0` and `""`. */ -export function getInvocationParams(tags?: string[] | Record): Record | undefined { - if (!tags || Array.isArray(tags)) { - return undefined; +const setIfDefined = (target: Record, key: string, value: unknown): void => { + if (value !== undefined && value !== null) target[key] = value as SpanAttributeValue; +}; + +/** + * Like `setIfDefined`, but converts the value with `Number()` and skips only when the + * result is `NaN`. This ensures numeric 0 makes it through (unlike truthy checks). + */ +const setNumberIfDefined = (target: Record, key: string, value: unknown): void => { + const n = Number(value); + if (!Number.isNaN(n)) target[key] = n; +}; + +/** + * Converts a value to a string. Avoids double-quoted JSON strings where a plain + * string is desired, but still handles objects/arrays safely. + */ +function asString(v: unknown): string { + if (typeof v === 'string') return v; + try { + return JSON.stringify(v); + } catch { + return String(v); } - return tags.invocation_params as Record | undefined; } /** - * Normalize a single message role to standard gen_ai format + * Normalizes a single role token to our canonical set. + * + * @param role Incoming role value (free-form, any casing) + * @returns Canonical role: 'user' | 'assistant' | 'system' | 'function' | 'tool' | */ -export function normalizeMessageRole(role: string): string { - const roleMap: Record = { - human: 'user', - ai: 'assistant', - system: 'system', - function: 'function', - tool: 'tool', - }; - - const normalizedRole = role.toLowerCase(); - return roleMap[normalizedRole] || normalizedRole; +function normalizeMessageRole(role: string): string { + const normalized = role.toLowerCase(); + return ROLE_MAP[normalized] ?? normalized; } /** - * Extract role from constructor name + * Infers a role from a LangChain message constructor name. + * + * Checks for substrings like "System", "Human", "AI", etc. */ -export function normalizeRoleName(roleName: string): string { - if (roleName.includes('System')) { - return 'system'; - } - if (roleName.includes('Human')) { - return 'user'; - } - if (roleName.includes('AI') || roleName.includes('Assistant')) { - return 'assistant'; - } - if (roleName.includes('Function')) { - return 'function'; - } - if (roleName.includes('Tool')) { - return 'tool'; - } +function normalizeRoleNameFromCtor(name: string): string { + if (name.includes('System')) return 'system'; + if (name.includes('Human')) return 'user'; + if (name.includes('AI') || name.includes('Assistant')) return 'assistant'; + if (name.includes('Function')) return 'function'; + if (name.includes('Tool')) return 'tool'; return 'user'; } /** - * Normalize LangChain messages to simple {role, content} format - * Handles both raw message objects and serialized LangChain message format + * Returns invocation params from a LangChain `tags` object. + * + * LangChain often passes runtime parameters (model, temperature, etc.) via the + * `tags.invocation_params` bag. If `tags` is an array (LangChain sometimes uses + * string tags), we return `undefined`. + * + * @param tags LangChain tags (string[] or record) + * @returns The `invocation_params` object, if present + */ +export function getInvocationParams(tags?: string[] | Record): Record | undefined { + if (!tags || Array.isArray(tags)) return undefined; + return tags.invocation_params as Record | undefined; +} + +/** + * Normalizes a heterogeneous set of LangChain messages to `{ role, content }`. + * + * Why so many branches? LangChain messages can arrive in several shapes: + * - Message classes with `_getType()` (most reliable) + * - Classes with meaningful constructor names (e.g. `SystemMessage`) + * - Plain objects with `type`, or `{ role, content }` + * - Serialized format with `{ lc: 1, id: [...], kwargs: { content } }` + * We preserve the prioritization to minimize behavioral drift. + * + * @param messages Mixed LangChain messages + * @returns Array of normalized `{ role, content }` */ export function normalizeLangChainMessages(messages: LangChainMessage[]): Array<{ role: string; content: string }> { return messages.map(message => { - // First, try to get the message type from _getType() method (most reliable) - if (typeof (message as { _getType?: () => string })._getType === 'function') { - const messageType = (message as { _getType: () => string })._getType(); + // 1) Prefer _getType() when present + const maybeGetType = (message as { _getType?: () => string })._getType; + if (typeof maybeGetType === 'function') { + const messageType = maybeGetType.call(message); return { role: normalizeMessageRole(messageType), - content: String((message as { content?: string }).content || ''), + content: asString(message.content), }; } - // Check constructor name (for LangChain message objects like SystemMessage, HumanMessage, etc.) - const constructorName = message.constructor?.name; - if (constructorName) { - const role = normalizeRoleName(constructorName); + // 2) Then try constructor name (SystemMessage / HumanMessage / ...) + const ctor = (message as { constructor?: { name?: string } }).constructor?.name; + if (ctor) { return { - role: normalizeMessageRole(role), - content: String((message as { content?: string }).content || ''), + role: normalizeMessageRole(normalizeRoleNameFromCtor(ctor)), + content: asString(message.content), }; } - // Handle message objects with type field + // 3) Then objects with `type` if (message.type) { const role = String(message.type).toLowerCase(); return { role: normalizeMessageRole(role), - content: String(message.content || ''), + content: asString(message.content), }; } - // Handle regular message objects with role and content - if (message.role && message.content) { + // 4) Then objects with `{ role, content }` + if (message.role) { return { role: normalizeMessageRole(String(message.role)), - content: String(message.content), + content: asString(message.content), }; } - // Handle LangChain serialized format with lc: 1 + // 5) Serialized LangChain format (lc: 1) if (message.lc === 1 && message.kwargs) { - // Extract role from the message type (e.g., HumanMessage -> user) - const messageType = Array.isArray(message.id) && message.id.length > 0 ? message.id[message.id.length - 1] : ''; - const role = typeof messageType === 'string' ? normalizeRoleName(messageType) : 'user'; + const id = message.id; + const messageType = Array.isArray(id) && id.length > 0 ? id[id.length - 1] : ''; + const role = typeof messageType === 'string' ? normalizeRoleNameFromCtor(messageType) : 'user'; return { role: normalizeMessageRole(role), - content: String(message.kwargs.content || ''), + content: asString(message.kwargs?.content), }; } - // Fallback: return as-is if we can't normalize + // 6) Fallback: treat as user text return { role: 'user', - content: String(message.content || JSON.stringify(message)), + content: asString(message.content), }; }); } /** - * Extract common request attributes shared by LLM and chat model requests + * Extracts request attributes common to both LLM and ChatModel invocations. + * + * Source precedence: + * 1) `invocationParams` (highest) + * 2) `langSmithMetadata` + * 3) `serialized.kwargs` + * + * Numeric values are set even when 0 (e.g. `temperature: 0`), but skipped if `NaN`. */ -export function extractCommonRequestAttributes( +function extractCommonRequestAttributes( serialized: LangChainSerializedLLM, invocationParams?: Record, langSmithMetadata?: Record, ): Record { - const attributes: Record = {}; + const attrs: Record = {}; - // Priority: invocationParams > LangSmith > kwargs const temperature = invocationParams?.temperature ?? langSmithMetadata?.ls_temperature ?? serialized.kwargs?.temperature; - if (Number(temperature)) { - attributes[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE] = Number(temperature); - } + setNumberIfDefined(attrs, GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE, temperature); const maxTokens = invocationParams?.max_tokens ?? langSmithMetadata?.ls_max_tokens ?? serialized.kwargs?.max_tokens; - if (Number(maxTokens)) { - attributes[GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE] = Number(maxTokens); - } + setNumberIfDefined(attrs, GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE, maxTokens); const topP = invocationParams?.top_p ?? serialized.kwargs?.top_p; - if (Number(topP)) { - attributes[GEN_AI_REQUEST_TOP_P_ATTRIBUTE] = Number(topP); - } + setNumberIfDefined(attrs, GEN_AI_REQUEST_TOP_P_ATTRIBUTE, topP); const frequencyPenalty = invocationParams?.frequency_penalty ?? serialized.kwargs?.frequency_penalty; - if (Number(frequencyPenalty)) { - attributes[GEN_AI_REQUEST_FREQUENCY_PENALTY_ATTRIBUTE] = Number(frequencyPenalty); - } + setNumberIfDefined(attrs, GEN_AI_REQUEST_FREQUENCY_PENALTY_ATTRIBUTE, frequencyPenalty); const presencePenalty = invocationParams?.presence_penalty ?? serialized.kwargs?.presence_penalty; - if (Number(presencePenalty)) { - attributes[GEN_AI_REQUEST_PRESENCE_PENALTY_ATTRIBUTE] = Number(presencePenalty); - } + setNumberIfDefined(attrs, GEN_AI_REQUEST_PRESENCE_PENALTY_ATTRIBUTE, presencePenalty); - const streaming = invocationParams?.stream; - if (streaming !== undefined && streaming !== null) { - // Sometimes this is set to false even for stream requests - // This issue stems from LangChain's callback handler and should be investigated - attributes[GEN_AI_REQUEST_STREAM_ATTRIBUTE] = Boolean(streaming); + // LangChain uses `stream`. We only set the attribute if the key actually exists + // (some callbacks report `false` even on streamed requests, this stems from LangChain's callback handler). + if (invocationParams && 'stream' in invocationParams) { + setIfDefined(attrs, GEN_AI_REQUEST_STREAM_ATTRIBUTE, Boolean(invocationParams.stream)); } - return attributes; + return attrs; } /** - * Extract request attributes from LLM start event + * Small helper to assemble boilerplate attributes shared by both request extractors. + */ +function baseRequestAttributes( + system: unknown, + modelName: unknown, + operation: 'pipeline' | 'chat', + serialized: LangChainSerializedLLM, + invocationParams?: Record, + langSmithMetadata?: Record, +): Record { + return { + [GEN_AI_SYSTEM_ATTRIBUTE]: asString(system ?? 'langchain'), + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: operation, + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: asString(modelName), + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: LANGCHAIN_ORIGIN, + ...extractCommonRequestAttributes(serialized, invocationParams, langSmithMetadata), + }; +} + +/** + * Extracts attributes for plain LLM invocations (string prompts). + * + * - Operation is tagged as `pipeline` to distinguish from chat-style invocations. + * - When `recordInputs` is true, string prompts are wrapped into `{role:"user"}` + * messages to align with the chat schema used elsewhere. */ export function extractLLMRequestAttributes( serialized: LangChainSerializedLLM, @@ -190,29 +244,26 @@ export function extractLLMRequestAttributes( invocationParams?: Record, langSmithMetadata?: Record, ): Record { - const system = JSON.stringify(langSmithMetadata?.ls_provider); - const modelName = JSON.stringify(invocationParams?.model ?? langSmithMetadata?.ls_model_name ?? 'unknown'); + const system = langSmithMetadata?.ls_provider; + const modelName = invocationParams?.model ?? langSmithMetadata?.ls_model_name ?? 'unknown'; - const attributes: Record = { - [GEN_AI_SYSTEM_ATTRIBUTE]: system, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'pipeline', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: modelName, - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: LANGCHAIN_ORIGIN, - ...extractCommonRequestAttributes(serialized, invocationParams, langSmithMetadata), - }; + const attrs = baseRequestAttributes(system, modelName, 'pipeline', serialized, invocationParams, langSmithMetadata); - // Add prompts if recordInputs is enabled - if (recordInputs && prompts && prompts.length > 0) { - // Convert string prompts to message format - const messages = prompts.map(prompt => ({ role: 'user', content: prompt })); - attributes[GEN_AI_REQUEST_MESSAGES_ATTRIBUTE] = JSON.stringify(messages); + if (recordInputs && Array.isArray(prompts) && prompts.length > 0) { + const messages = prompts.map(p => ({ role: 'user', content: p })); + setIfDefined(attrs, GEN_AI_REQUEST_MESSAGES_ATTRIBUTE, asString(messages)); } - return attributes; + return attrs; } /** - * Extract request attributes from chat model start event + * Extracts attributes for ChatModel invocations (array-of-arrays of messages). + * + * - Operation is tagged as `chat`. + * - We flatten LangChain's `LangChainMessage[][]` and normalize shapes into a + * consistent `{ role, content }` array when `recordInputs` is true. + * - Provider system value falls back to `serialized.id?.[2]`. */ export function extractChatModelRequestAttributes( serialized: LangChainSerializedLLM, @@ -221,68 +272,60 @@ export function extractChatModelRequestAttributes( invocationParams?: Record, langSmithMetadata?: Record, ): Record { - // Provider either exists in LangSmith metadata or is the 3rd index of the id array of the serialized LLM - const system = JSON.stringify(langSmithMetadata?.ls_provider ?? serialized.id?.[2]); - const modelName = JSON.stringify(invocationParams?.model ?? langSmithMetadata?.ls_model_name ?? 'unknown'); - - const attributes: Record = { - [GEN_AI_SYSTEM_ATTRIBUTE]: system, - [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'chat', - [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: modelName, - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: LANGCHAIN_ORIGIN, - ...extractCommonRequestAttributes(serialized, invocationParams, langSmithMetadata), - }; + const system = langSmithMetadata?.ls_provider ?? serialized.id?.[2]; + const modelName = invocationParams?.model ?? langSmithMetadata?.ls_model_name ?? 'unknown'; + + const attrs = baseRequestAttributes(system, modelName, 'chat', serialized, invocationParams, langSmithMetadata); - // Add messages if recordInputs is enabled - if (recordInputs && messages && messages.length > 0) { - // Flatten the messages array (LangChain passes array of message arrays) - const flatMessages = messages.flat(); - // Normalize messages to extract content from LangChain serialized format - const normalizedMessages = normalizeLangChainMessages(flatMessages); - attributes[GEN_AI_REQUEST_MESSAGES_ATTRIBUTE] = JSON.stringify(normalizedMessages); + if (recordInputs && Array.isArray(messages) && messages.length > 0) { + const normalized = normalizeLangChainMessages(messages.flat()); + setIfDefined(attrs, GEN_AI_REQUEST_MESSAGES_ATTRIBUTE, asString(normalized)); } - return attributes; + return attrs; } /** - * Extract tool calls attributes from generations + * Scans generations for Anthropic-style `tool_use` items and records them. + * + * LangChain represents some provider messages (e.g., Anthropic) with a `message.content` + * array that may include objects `{ type: 'tool_use', ... }`. We collect and attach + * them as a JSON array on `gen_ai.response.tool_calls` for downstream consumers. */ -export function extractToolCallsAttributes(generations: LangChainMessage[][], span: Span): void { - const toolCalls: Array = []; - - // Flatten the generations array (LangChain returns [[generation]]) +function addToolCallsAttributes(generations: LangChainMessage[][], attrs: Record): void { + const toolCalls: unknown[] = []; const flatGenerations = generations.flat(); for (const gen of flatGenerations) { - // Check if message has a content array (Anthropic format) - if (gen.message?.content && Array.isArray(gen.message.content)) { - for (const contentItem of gen.message.content) { - const typedContent = contentItem as { type?: string; id?: string; name?: string; input?: unknown }; - if (typedContent.type === 'tool_use') { - toolCalls.push(typedContent); - } + const content = gen.message?.content; + if (Array.isArray(content)) { + for (const item of content) { + const t = item as { type: string }; + if (t.type === 'tool_use') toolCalls.push(t); } } } if (toolCalls.length > 0) { - span.setAttributes({ - [GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE]: JSON.stringify(toolCalls), - }); + setIfDefined(attrs, GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE, asString(toolCalls)); } } /** - * Extract token usage attributes from LLM output + * Adds token usage attributes, supporting both OpenAI (`tokenUsage`) and Anthropic (`usage`) formats. + * - Preserve zero values (0 tokens) by avoiding truthy checks. + * - Compute a total for Anthropic when not explicitly provided. + * - Include cache token metrics when present. */ -export function extractTokenUsageAttributes(llmOutput: LangChainLLMResult['llmOutput'], span: Span): void { +function addTokenUsageAttributes( + llmOutput: LangChainLLMResult['llmOutput'], + attrs: Record, +): void { if (!llmOutput) return; - // Try standard tokenUsage format (OpenAI) + const tokenUsage = llmOutput.tokenUsage as | { promptTokens?: number; completionTokens?: number; totalTokens?: number } | undefined; - // Try Anthropic format (usage.input_tokens, etc.) const anthropicUsage = llmOutput.usage as | { input_tokens?: number; @@ -293,91 +336,85 @@ export function extractTokenUsageAttributes(llmOutput: LangChainLLMResult['llmOu | undefined; if (tokenUsage) { - if (Number(tokenUsage.promptTokens)) { - span.setAttributes({ [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: tokenUsage.promptTokens }); - } - if (Number(tokenUsage.completionTokens)) { - span.setAttributes({ [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: tokenUsage.completionTokens }); - } - if (Number(tokenUsage.totalTokens)) { - span.setAttributes({ [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: tokenUsage.totalTokens }); - } + setNumberIfDefined(attrs, GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE, tokenUsage.promptTokens); + setNumberIfDefined(attrs, GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, tokenUsage.completionTokens); + setNumberIfDefined(attrs, GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, tokenUsage.totalTokens); } else if (anthropicUsage) { - // Handle Anthropic format - if (Number(anthropicUsage.input_tokens)) { - span.setAttributes({ [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: anthropicUsage.input_tokens }); - } - if (Number(anthropicUsage.output_tokens)) { - span.setAttributes({ [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: anthropicUsage.output_tokens }); - } - // Calculate total tokens for Anthropic - const total = (anthropicUsage.input_tokens || 0) + (anthropicUsage.output_tokens || 0); - if (total > 0) { - span.setAttributes({ [GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: total }); - } - // Add cache tokens if present - if (anthropicUsage.cache_creation_input_tokens !== undefined) { - span.setAttributes({ 'gen_ai.usage.cache_creation_input_tokens': anthropicUsage.cache_creation_input_tokens }); - } - if (anthropicUsage.cache_read_input_tokens !== undefined) { - span.setAttributes({ 'gen_ai.usage.cache_read_input_tokens': anthropicUsage.cache_read_input_tokens }); - } + setNumberIfDefined(attrs, GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE, anthropicUsage.input_tokens); + setNumberIfDefined(attrs, GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE, anthropicUsage.output_tokens); + + // Compute total when not provided by the provider. + const input = Number(anthropicUsage.input_tokens); + const output = Number(anthropicUsage.output_tokens); + const total = (Number.isNaN(input) ? 0 : input) + (Number.isNaN(output) ? 0 : output); + if (total > 0) setNumberIfDefined(attrs, GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, total); + + // Extra Anthropic cache metrics (present only when caching is enabled) + if (anthropicUsage.cache_creation_input_tokens !== undefined) + setNumberIfDefined( + attrs, + GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS_ATTRIBUTE, + anthropicUsage.cache_creation_input_tokens, + ); + if (anthropicUsage.cache_read_input_tokens !== undefined) + setNumberIfDefined(attrs, GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS_ATTRIBUTE, anthropicUsage.cache_read_input_tokens); } } /** - * Add response attributes from LLM result + * Extracts response-related attributes based on a `LangChainLLMResult`. + * + * - Records finish reasons when present on generations (e.g., OpenAI) + * - When `recordOutputs` is true, captures textual response content and any + * tool calls. + * - Also propagates model name (`model_name` or `model`), response `id`, and + * `stop_reason` (for providers that use it). */ -export function addLLMResponseAttributes(span: Span, response: LangChainLLMResult, recordOutputs: boolean): void { +export function extractLlmResponseAttributes( + response: LangChainLLMResult, + recordOutputs: boolean, +): Record | undefined { if (!response) return; - // Extract finish reasons - if (response.generations && Array.isArray(response.generations)) { + const attrs: Record = {}; + + if (Array.isArray(response.generations)) { const finishReasons = response.generations - .map(gen => gen.generation_info?.finish_reason) - .filter(reason => typeof reason === 'string'); + .map(g => g.generation_info?.finish_reason) + .filter((r): r is string => typeof r === 'string'); if (finishReasons.length > 0) { - span.setAttributes({ - [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: JSON.stringify(finishReasons), - }); + setIfDefined(attrs, GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE, asString(finishReasons)); } - // Extract response text if recordOutputs is enabled if (recordOutputs) { - const responseTexts = response.generations + const texts = response.generations .flat() - .map(gen => gen.text || gen.message?.content) - .filter(text => typeof text === 'string'); + .map(gen => gen.text ?? gen.message?.content) + .filter(t => typeof t === 'string'); - if (responseTexts.length > 0) { - span.setAttributes({ - [GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: JSON.stringify(responseTexts), - }); + if (texts.length > 0) { + setIfDefined(attrs, GEN_AI_RESPONSE_TEXT_ATTRIBUTE, asString(texts)); } - // Extract tool calls from message content if present - extractToolCallsAttributes(response.generations as LangChainMessage[][], span); + addToolCallsAttributes(response.generations as LangChainMessage[][], attrs); } } - // Extract token usage - handle both formats (OpenAI and Anthropic) - extractTokenUsageAttributes(response.llmOutput, span); + addTokenUsageAttributes(response.llmOutput, attrs); - // Extract model name from response (handle both model_name and model fields) - const modelName = response.llmOutput?.model_name || response.llmOutput?.model; + const llmOutput = response.llmOutput as { model_name?: string; model?: string; id?: string; stop_reason?: string }; + // Provider model identifier: `model_name` (OpenAI-style) or `model` (others) + const modelName = llmOutput?.model_name ?? llmOutput?.model; + if (modelName) setIfDefined(attrs, GEN_AI_RESPONSE_MODEL_ATTRIBUTE, modelName); - if (modelName) { - span.setAttributes({ [GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: JSON.stringify(modelName) }); + if (llmOutput?.id) { + setIfDefined(attrs, GEN_AI_RESPONSE_ID_ATTRIBUTE, llmOutput.id); } - // Add response ID (useful for debugging/correlation) - if (response.llmOutput?.id) { - span.setAttributes({ 'gen_ai.response.id': JSON.stringify(response.llmOutput.id) }); + if (llmOutput?.stop_reason) { + setIfDefined(attrs, GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE, asString(llmOutput.stop_reason)); } - // Add stop reason as finish reason if not already captured - if (response.llmOutput?.stop_reason) { - span.setAttributes({ 'gen_ai.response.stop_reason': JSON.stringify(response.llmOutput.stop_reason) }); - } + return attrs; } diff --git a/packages/node/src/integrations/tracing/langchain/index.ts b/packages/node/src/integrations/tracing/langchain/index.ts index 2319487883ab..c88f85db926a 100644 --- a/packages/node/src/integrations/tracing/langchain/index.ts +++ b/packages/node/src/integrations/tracing/langchain/index.ts @@ -100,10 +100,9 @@ const _langChainIntegration = ((options: LangChainOptions = {}) => { * ## Supported Events * * The integration captures the following LangChain lifecycle events: - * - LLM/Chat Model: start, new token (streaming), end, error + * - LLM/Chat Model: start, end, error * - Chain: start, end, error * - Tool: start, end, error - * - Retriever: start, end, error * */ export const langChainIntegration = defineIntegration(_langChainIntegration); diff --git a/packages/node/src/integrations/tracing/langchain/instrumentation.ts b/packages/node/src/integrations/tracing/langchain/instrumentation.ts index f6a015019f26..bb3f6bd15563 100644 --- a/packages/node/src/integrations/tracing/langchain/instrumentation.ts +++ b/packages/node/src/integrations/tracing/langchain/instrumentation.ts @@ -171,10 +171,23 @@ export class SentryLangChainInstrumentation extends InstrumentationBase typeof exp === 'function' && exp.name?.includes('Chat'), - ) as { prototype: unknown; name: string } | undefined; + // Known chat model class names for each provider + const knownChatModelNames = [ + 'ChatAnthropic', + 'ChatOpenAI', + 'ChatGoogleGenerativeAI', + 'ChatMistralAI', + 'ChatVertexAI', + 'ChatGroq', + ]; + + // Find a chat model class in the exports by checking known class names + const chatModelClass = Object.values(exports).find(exp => { + if (typeof exp !== 'function') { + return false; + } + return knownChatModelNames.includes(exp.name); + }) as { prototype: unknown; name: string } | undefined; if (!chatModelClass) { return; From ab4350e8e240026856f8ca1fce2430b22f8735d0 Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Fri, 17 Oct 2025 11:43:52 +0200 Subject: [PATCH 03/12] fix ci failing --- packages/core/src/utils/langchain/utils.ts | 3 ++- yarn.lock | 14 ++------------ 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/packages/core/src/utils/langchain/utils.ts b/packages/core/src/utils/langchain/utils.ts index 9c0024e632eb..54198985b0ee 100644 --- a/packages/core/src/utils/langchain/utils.ts +++ b/packages/core/src/utils/langchain/utils.ts @@ -1,4 +1,5 @@ -import { type SpanAttributeValue, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '../..'; +import { SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '../../semanticAttributes'; +import type { SpanAttributeValue } from '../../types-hoist/span'; import { GEN_AI_OPERATION_NAME_ATTRIBUTE, GEN_AI_REQUEST_FREQUENCY_PENALTY_ATTRIBUTE, diff --git a/yarn.lock b/yarn.lock index 0aee43c39414..41f079f365fa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -32054,27 +32054,17 @@ zip-stream@^6.0.1: compress-commons "^6.0.2" readable-stream "^4.0.0" -zod-to-json-schema@^3.22.3: +zod-to-json-schema@^3.22.3, zod-to-json-schema@^3.24.1: version "3.24.6" resolved "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.6.tgz#5920f020c4d2647edfbb954fa036082b92c9e12d" integrity sha512-h/z3PKvcTcTetyjl1fkj79MHNEjm+HpD6NXheWjzOekY7kV+lwDYnHw+ivHkijnCSMz1yJaWBD9vu/Fcmk+vEg== -zod-to-json-schema@^3.24.1: - version "3.24.5" - resolved "https://registry.yarnpkg.com/zod-to-json-schema/-/zod-to-json-schema-3.24.5.tgz#d1095440b147fb7c2093812a53c54df8d5df50a3" - integrity sha512-/AuWwMP+YqiPbsJx5D6TfgRTc4kTLjsh5SOcd4bLsfUg2RcEXrFMJl1DGgdHy2aCfsIA/cr/1JM0xcB2GZji8g== - zod@3.22.3: version "3.22.3" resolved "https://registry.yarnpkg.com/zod/-/zod-3.22.3.tgz#2fbc96118b174290d94e8896371c95629e87a060" integrity sha512-EjIevzuJRiRPbVH4mGc8nApb/lVLKVpmUhAaR5R5doKGfAnGJ6Gr3CViAVjP+4FWSxCsybeWQdcgCtbX+7oZug== -zod@^3.22.2, zod@^3.22.4, zod@^3.23.8, zod@^3.24.1: - version "3.25.75" - resolved "https://registry.yarnpkg.com/zod/-/zod-3.25.75.tgz#8ff9be2fbbcb381a9236f9f74a8879ca29dcc504" - integrity sha512-OhpzAmVzabPOL6C3A3gpAifqr9MqihV/Msx3gor2b2kviCgcb+HM9SEOpMWwwNp9MRunWnhtAKUoo0AHhjyPPg== - -zod@^3.25.32: +zod@^3.22.2, zod@^3.22.4, zod@^3.23.8, zod@^3.24.1, zod@^3.25.32: version "3.25.76" resolved "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz#26841c3f6fd22a6a2760e7ccb719179768471e34" integrity sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ== From e61c71041d1076af60dee87d1d67b249e4284617 Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Fri, 17 Oct 2025 12:14:48 +0200 Subject: [PATCH 04/12] add missing imports --- packages/astro/src/index.server.ts | 1 + packages/aws-serverless/src/index.ts | 1 + packages/bun/src/index.ts | 1 + packages/google-cloud-serverless/src/index.ts | 1 + 4 files changed, 4 insertions(+) diff --git a/packages/astro/src/index.server.ts b/packages/astro/src/index.server.ts index b617f5010a3d..69ca79e04a17 100644 --- a/packages/astro/src/index.server.ts +++ b/packages/astro/src/index.server.ts @@ -31,6 +31,7 @@ export { contextLinesIntegration, continueTrace, createGetModuleFromFilename, + createLangChainCallbackHandler, createTransport, cron, dataloaderIntegration, diff --git a/packages/aws-serverless/src/index.ts b/packages/aws-serverless/src/index.ts index ed66b3c99ee0..da0393d9b0e9 100644 --- a/packages/aws-serverless/src/index.ts +++ b/packages/aws-serverless/src/index.ts @@ -42,6 +42,7 @@ export { close, getSentryRelease, createGetModuleFromFilename, + createLangChainCallbackHandler, httpHeadersToSpanAttributes, winterCGHeadersToDict, // eslint-disable-next-line deprecation/deprecation diff --git a/packages/bun/src/index.ts b/packages/bun/src/index.ts index 4c503c1fb6c1..33af15790191 100644 --- a/packages/bun/src/index.ts +++ b/packages/bun/src/index.ts @@ -62,6 +62,7 @@ export { close, getSentryRelease, createGetModuleFromFilename, + createLangChainCallbackHandler, httpHeadersToSpanAttributes, winterCGHeadersToDict, // eslint-disable-next-line deprecation/deprecation diff --git a/packages/google-cloud-serverless/src/index.ts b/packages/google-cloud-serverless/src/index.ts index 3213abd8ada5..02e55c45a7ba 100644 --- a/packages/google-cloud-serverless/src/index.ts +++ b/packages/google-cloud-serverless/src/index.ts @@ -42,6 +42,7 @@ export { close, getSentryRelease, createGetModuleFromFilename, + createLangChainCallbackHandler, httpHeadersToSpanAttributes, winterCGHeadersToDict, // eslint-disable-next-line deprecation/deprecation From 630dfb2ea74ced1599a4b57814d884c0b770abd7 Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Fri, 17 Oct 2025 12:40:15 +0200 Subject: [PATCH 05/12] more tests --- .../tracing/langchain/scenario-tools.mjs | 91 +++++++++++++++++++ .../suites/tracing/langchain/scenario.mjs | 2 + .../suites/tracing/langchain/test.ts | 34 +++++++ packages/core/src/utils/langchain/utils.ts | 5 +- 4 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/langchain/scenario-tools.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-tools.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-tools.mjs new file mode 100644 index 000000000000..d14da5c5ad48 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-tools.mjs @@ -0,0 +1,91 @@ +import { ChatAnthropic } from '@langchain/anthropic'; +import * as Sentry from '@sentry/node'; +import express from 'express'; + +function startMockAnthropicServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1/messages', (req, res) => { + const model = req.body.model; + + // Simulate tool call response + res.json({ + id: 'msg_tool_test_123', + type: 'message', + role: 'assistant', + model: model, + content: [ + { + type: 'text', + text: 'Let me check the weather for you.', + }, + { + type: 'tool_use', + id: 'toolu_01A09q90qw90lq917835lq9', + name: 'get_weather', + input: { location: 'San Francisco, CA' }, + }, + { + type: 'text', + text: 'The weather looks great!', + }, + ], + stop_reason: 'tool_use', + stop_sequence: null, + usage: { + input_tokens: 20, + output_tokens: 30, + }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockAnthropicServer(); + const baseUrl = `http://localhost:${server.address().port}`; + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const model = new ChatAnthropic({ + model: 'claude-3-5-sonnet-20241022', + temperature: 0.7, + maxTokens: 150, + apiKey: 'mock-api-key', + clientOptions: { + baseURL: baseUrl, + }, + }); + + await model.invoke('What is the weather in San Francisco?', { + tools: [ + { + name: 'get_weather', + description: 'Get the current weather in a given location', + input_schema: { + type: 'object', + properties: { + location: { + type: 'string', + description: 'The city and state, e.g. San Francisco, CA', + }, + }, + required: ['location'], + }, + }, + ], + }); + }); + + await Sentry.flush(2000); + + server.close(); +} + +run(); + diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/scenario.mjs index 132e519c0b8c..44b4175c5004 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/scenario.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/scenario.mjs @@ -102,6 +102,8 @@ async function run() { } }); + await Sentry.flush(2000); + server.close(); } diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts index 357b50e78488..e3738b61b7a7 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/test.ts @@ -160,4 +160,38 @@ describe('LangChain integration', () => { .completed(); }); }); + + const EXPECTED_TRANSACTION_TOOL_CALLS = { + transaction: 'main', + spans: expect.arrayContaining([ + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'auto.ai.langchain', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-3-5-sonnet-20241022', + 'gen_ai.request.temperature': 0.7, + 'gen_ai.request.max_tokens': 150, + 'gen_ai.usage.input_tokens': 20, + 'gen_ai.usage.output_tokens': 30, + 'gen_ai.usage.total_tokens': 50, + 'gen_ai.response.id': expect.any(String), + 'gen_ai.response.model': expect.any(String), + 'gen_ai.response.stop_reason': 'tool_use', + 'gen_ai.response.tool_calls': expect.any(String), + }), + description: 'chat claude-3-5-sonnet-20241022', + op: 'gen_ai.chat', + origin: 'auto.ai.langchain', + status: 'ok', + }), + ]), + }; + + createEsmAndCjsTests(__dirname, 'scenario-tools.mjs', 'instrument.mjs', (createRunner, test) => { + test('creates langchain spans with tool calls', async () => { + await createRunner().ignore('event').expect({ transaction: EXPECTED_TRANSACTION_TOOL_CALLS }).start().completed(); + }); + }); }); diff --git a/packages/core/src/utils/langchain/utils.ts b/packages/core/src/utils/langchain/utils.ts index 54198985b0ee..236216603e38 100644 --- a/packages/core/src/utils/langchain/utils.ts +++ b/packages/core/src/utils/langchain/utils.ts @@ -388,6 +388,9 @@ export function extractLlmResponseAttributes( setIfDefined(attrs, GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE, asString(finishReasons)); } + // Tool calls metadata (names, IDs) are not PII, so capture them regardless of recordOutputs + addToolCallsAttributes(response.generations as LangChainMessage[][], attrs); + if (recordOutputs) { const texts = response.generations .flat() @@ -397,8 +400,6 @@ export function extractLlmResponseAttributes( if (texts.length > 0) { setIfDefined(attrs, GEN_AI_RESPONSE_TEXT_ATTRIBUTE, asString(texts)); } - - addToolCallsAttributes(response.generations as LangChainMessage[][], attrs); } } From abe63cf3a69b83cec619e499c17fe9c083b3251d Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Fri, 17 Oct 2025 12:40:54 +0200 Subject: [PATCH 06/12] bump bundle size --- .size-limit.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.size-limit.js b/.size-limit.js index 5de4268a53d6..b805d114bdff 100644 --- a/.size-limit.js +++ b/.size-limit.js @@ -233,7 +233,7 @@ module.exports = [ import: createImport('init'), ignore: [...builtinModules, ...nodePrefixedBuiltinModules], gzip: true, - limit: '156 KB', + limit: '157 KB', }, { name: '@sentry/node - without tracing', From 459ab359e62396d946f0380be211f888397307f8 Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Fri, 17 Oct 2025 12:46:38 +0200 Subject: [PATCH 07/12] nice catch cursor --- .../suites/tracing/langchain/scenario-tools.mjs | 1 - .../node/src/integrations/tracing/langchain/instrumentation.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-tools.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-tools.mjs index d14da5c5ad48..256ee4568884 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-tools.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/scenario-tools.mjs @@ -88,4 +88,3 @@ async function run() { } run(); - diff --git a/packages/node/src/integrations/tracing/langchain/instrumentation.ts b/packages/node/src/integrations/tracing/langchain/instrumentation.ts index bb3f6bd15563..f171a2dfb022 100644 --- a/packages/node/src/integrations/tracing/langchain/instrumentation.ts +++ b/packages/node/src/integrations/tracing/langchain/instrumentation.ts @@ -108,7 +108,7 @@ export class SentryLangChainInstrumentation extends InstrumentationBase Date: Fri, 17 Oct 2025 15:47:25 +0200 Subject: [PATCH 08/12] add op to attributes --- packages/core/src/utils/langchain/index.ts | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/packages/core/src/utils/langchain/index.ts b/packages/core/src/utils/langchain/index.ts index 81e3f8eaf072..7866722ed4b0 100644 --- a/packages/core/src/utils/langchain/index.ts +++ b/packages/core/src/utils/langchain/index.ts @@ -1,5 +1,5 @@ import { captureException } from '../../exports'; -import { SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '../../semanticAttributes'; +import { SEMANTIC_ATTRIBUTE_SENTRY_OP, SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN } from '../../semanticAttributes'; import { SPAN_STATUS_ERROR } from '../../tracing'; import { startSpanManual } from '../../tracing/trace'; import type { Span, SpanAttributeValue } from '../../types-hoist/span'; @@ -97,7 +97,10 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): { name: `${operationName} ${modelName}`, op: 'gen_ai.pipeline', - attributes, + attributes: { + ...attributes, + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.pipeline', + }, }, span => { spanMap.set(runId, span); @@ -130,7 +133,10 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): { name: `${operationName} ${modelName}`, op: 'gen_ai.chat', - attributes, + attributes: { + ...attributes, + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', + }, }, span => { spanMap.set(runId, span); @@ -204,7 +210,10 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): { name: `chain ${chainName}`, op: 'gen_ai.invoke_agent', - attributes, + attributes: { + ...attributes, + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', + }, }, span => { spanMap.set(runId, span); @@ -273,7 +282,10 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): { name: `execute_tool ${toolName}`, op: 'gen_ai.execute_tool', - attributes, + attributes: { + ...attributes, + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.execute_tool', + }, }, span => { spanMap.set(runId, span); From 00292205898e6b992a891fd7257fedaa4d4c615f Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Tue, 21 Oct 2025 11:49:59 +0200 Subject: [PATCH 09/12] update with better types --- packages/core/src/index.ts | 2 +- packages/core/src/utils/langchain/index.ts | 128 +++++++++------ packages/core/src/utils/langchain/types.ts | 152 ++++++++++++++++-- packages/core/src/utils/langchain/utils.ts | 54 ++++--- .../integrations/tracing/langchain/index.ts | 3 +- 5 files changed, 241 insertions(+), 98 deletions(-) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 38ed0bab7f09..f3b29009b9ce 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -146,6 +146,7 @@ export { GOOGLE_GENAI_INTEGRATION_NAME } from './utils/google-genai/constants'; export type { GoogleGenAIResponse } from './utils/google-genai/types'; export { createLangChainCallbackHandler } from './utils/langchain'; export { LANGCHAIN_INTEGRATION_NAME } from './utils/langchain/constants'; +export type { LangChainOptions, LangChainIntegration } from './utils/langchain/types'; export type { OpenAiClient, OpenAiOptions, InstrumentedMethod } from './utils/openai/types'; export type { AnthropicAiClient, @@ -159,7 +160,6 @@ export type { GoogleGenAIOptions, GoogleGenAIIstrumentedMethod, } from './utils/google-genai/types'; -export type { LangChainOptions, LangChainIntegration } from './utils/langchain/types'; export type { FeatureFlag } from './utils/featureFlags'; export { diff --git a/packages/core/src/utils/langchain/index.ts b/packages/core/src/utils/langchain/index.ts index 7866722ed4b0..a34da24c197a 100644 --- a/packages/core/src/utils/langchain/index.ts +++ b/packages/core/src/utils/langchain/index.ts @@ -5,7 +5,13 @@ import { startSpanManual } from '../../tracing/trace'; import type { Span, SpanAttributeValue } from '../../types-hoist/span'; import { GEN_AI_OPERATION_NAME_ATTRIBUTE, GEN_AI_REQUEST_MODEL_ATTRIBUTE } from '../ai/gen-ai-attributes'; import { LANGCHAIN_ORIGIN } from './constants'; -import type { LangChainLLMResult, LangChainMessage, LangChainOptions, LangChainSerializedLLM } from './types'; +import type { + LangChainCallbackHandler, + LangChainLLMResult, + LangChainMessage, + LangChainOptions, + LangChainSerialized, +} from './types'; import { extractChatModelRequestAttributes, extractLLMRequestAttributes, @@ -19,42 +25,7 @@ import { * * This is a stateful handler that tracks spans across multiple LangChain executions. */ -export function createLangChainCallbackHandler(options: LangChainOptions = {}): { - handleLLMStart?: ( - llm: LangChainSerializedLLM, - prompts: string[], - runId: string, - parentRunId?: string, - extraParams?: Record, - tags?: string[] | Record, - metadata?: Record, - runName?: string | Record, - ) => void; - handleChatModelStart?: ( - llm: LangChainSerializedLLM, - messages: LangChainMessage[][], - runId: string, - parentRunId?: string, - extraParams?: Record, - tags?: string[] | Record, - metadata?: Record, - runName?: string | Record, - ) => void; - handleLLMNewToken?: (token: string, runId: string) => void; - handleLLMEnd?: (output: LangChainLLMResult, runId: string) => void; - handleLLMError?: (error: Error, runId: string) => void; - handleChainStart?: ( - chain: { name?: string }, - inputs: Record, - runId: string, - parentRunId?: string, - ) => void; - handleChainEnd?: (outputs: Record, runId: string) => void; - handleChainError?: (error: Error, runId: string) => void; - handleToolStart?: (tool: { name?: string }, input: string, runId: string, parentRunId?: string) => void; - handleToolEnd?: (output: string, runId: string) => void; - handleToolError?: (error: Error, runId: string) => void; -} { +export function createLangChainCallbackHandler(options: LangChainOptions = {}): LangChainCallbackHandler { const recordInputs = options.recordInputs ?? false; const recordOutputs = options.recordOutputs ?? false; @@ -76,20 +47,46 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): * Handler for LLM Start * This handler will be called by LangChain's callback handler when an LLM event is detected. */ - const handler = { + const handler: LangChainCallbackHandler = { + // Required LangChain BaseCallbackHandler properties + lc_serializable: false, + lc_namespace: ['langchain_core', 'callbacks', 'sentry'], + lc_secrets: undefined, + lc_attributes: undefined, + lc_aliases: undefined, + lc_serializable_keys: undefined, + lc_id: ['langchain_core', 'callbacks', 'sentry'], + lc_kwargs: {}, + name: 'SentryCallbackHandler', + + // BaseCallbackHandlerInput boolean flags + ignoreLLM: false, + ignoreChain: false, + ignoreAgent: false, + ignoreRetriever: false, + ignoreCustomEvent: false, + raiseError: false, + awaitHandlers: true, + handleLLMStart( - llm: LangChainSerializedLLM, + llm: unknown, prompts: string[], runId: string, _parentRunId?: string, _extraParams?: Record, - tags?: string[] | Record, + tags?: string[], metadata?: Record, - _runName?: string | Record, + _runName?: string, ) { try { const invocationParams = getInvocationParams(tags); - const attributes = extractLLMRequestAttributes(llm, prompts, recordInputs, invocationParams, metadata); + const attributes = extractLLMRequestAttributes( + llm as LangChainSerialized, + prompts, + recordInputs, + invocationParams, + metadata, + ); const modelName = attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]; const operationName = attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]; @@ -114,18 +111,24 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): // Chat Model Start Handler handleChatModelStart( - llm: LangChainSerializedLLM, - messages: LangChainMessage[][], + llm: unknown, + messages: unknown, runId: string, _parentRunId?: string, _extraParams?: Record, - tags?: string[] | Record, + tags?: string[], metadata?: Record, - _runName?: string | Record, + _runName?: string, ) { try { const invocationParams = getInvocationParams(tags); - const attributes = extractChatModelRequestAttributes(llm, messages, recordInputs, invocationParams, metadata); + const attributes = extractChatModelRequestAttributes( + llm as LangChainSerialized, + messages as LangChainMessage[][], + recordInputs, + invocationParams, + metadata, + ); const modelName = attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]; const operationName = attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]; @@ -151,7 +154,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): // LLM End Handler - note: handleLLMEnd with capital LLM (used by both LLMs and chat models!) handleLLMEnd( - output: LangChainLLMResult, + output: unknown, runId: string, _parentRunId?: string, _tags?: string[], @@ -160,7 +163,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): try { const span = spanMap.get(runId); if (span) { - const attributes = extractLlmResponseAttributes(output, recordOutputs); + const attributes = extractLlmResponseAttributes(output as LangChainLLMResult, recordOutputs); if (attributes) { span.setAttributes(attributes); } @@ -227,7 +230,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): }, // Chain End Handler - handleChainEnd(outputs: Record, runId: string) { + handleChainEnd(outputs: unknown, runId: string) { try { const span = spanMap.get(runId); if (span) { @@ -298,14 +301,14 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): }, // Tool End Handler - handleToolEnd(output: string, runId: string) { + handleToolEnd(output: unknown, runId: string) { try { const span = spanMap.get(runId); if (span) { // Add output if recordOutputs is enabled if (recordOutputs) { span.setAttributes({ - 'gen_ai.tool.output': output, + 'gen_ai.tool.output': JSON.stringify(output), }); } exitSpan(runId); @@ -334,6 +337,27 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): // silently ignore errors } }, + + // LangChain BaseCallbackHandler required methods + copy() { + return handler; + }, + + toJSON() { + return { + lc: 1, + type: 'not_implemented', + id: handler.lc_id, + }; + }, + + toJSONNotImplemented() { + return { + lc: 1, + type: 'not_implemented', + id: handler.lc_id, + }; + }, }; return handler; diff --git a/packages/core/src/utils/langchain/types.ts b/packages/core/src/utils/langchain/types.ts index 62bef5832ab7..e08542eefd60 100644 --- a/packages/core/src/utils/langchain/types.ts +++ b/packages/core/src/utils/langchain/types.ts @@ -16,18 +16,18 @@ export interface LangChainOptions { } /** - * LangChain LLM/Chat Model serialized data + * LangChain Serialized type (compatible with @langchain/core) + * Uses general types to be compatible with LangChain's Serialized interface. + * This is a flexible interface that accepts any serialized LangChain object. */ -export interface LangChainSerializedLLM { +export interface LangChainSerialized { [key: string]: unknown; - type?: string; lc?: number; + type?: string; id?: string[]; - kwargs?: { - [key: string]: unknown; - model?: string; - temperature?: number; - }; + name?: string; + graph?: Record; + kwargs?: Record; } /** @@ -60,16 +60,18 @@ export interface LangChainMessage { */ export interface LangChainLLMResult { [key: string]: unknown; - generations: Array<{ - text?: string; - message?: LangChainMessage; - generation_info?: { - [key: string]: unknown; + generations: Array< + Array<{ + text?: string; + message?: LangChainMessage; + generation_info?: { + [key: string]: unknown; - finish_reason?: string; - logprobs?: unknown; - }; - }>; + finish_reason?: string; + logprobs?: unknown; + }; + }> + >; llmOutput?: { [key: string]: unknown; tokenUsage?: { @@ -88,3 +90,119 @@ export interface LangChainIntegration { name: string; options: LangChainOptions; } + +/** + * LangChain callback handler interface + * Compatible with both BaseCallbackHandlerMethodsClass and BaseCallbackHandler from @langchain/core + * Uses general types and index signature for maximum compatibility across LangChain versions + */ +export interface LangChainCallbackHandler { + // Allow any additional properties for full compatibility + [key: string]: unknown; + + // LangChain BaseCallbackHandler class properties (matching the class interface exactly) + lc_serializable: boolean; + lc_namespace: ['langchain_core', 'callbacks', string]; + lc_secrets: { [key: string]: string } | undefined; + lc_attributes: { [key: string]: string } | undefined; + lc_aliases: { [key: string]: string } | undefined; + lc_serializable_keys: string[] | undefined; + lc_id: string[]; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + lc_kwargs: { [key: string]: any }; + name: string; + + // BaseCallbackHandlerInput properties (required boolean flags) + ignoreLLM: boolean; + ignoreChain: boolean; + ignoreAgent: boolean; + ignoreRetriever: boolean; + ignoreCustomEvent: boolean; + raiseError: boolean; + awaitHandlers: boolean; + + // Callback handler methods (properties with function signatures) + // Using 'any' for parameters and return types to match LangChain's BaseCallbackHandler exactly + handleLLMStart?: ( + llm: unknown, + prompts: string[], + runId: string, + parentRunId?: string, + extraParams?: Record, + tags?: string[], + metadata?: Record, + runName?: string, + ) => Promise | unknown; + handleChatModelStart?: ( + llm: unknown, + messages: unknown, + runId: string, + parentRunId?: string, + extraParams?: Record, + tags?: string[], + metadata?: Record, + runName?: string, + ) => Promise | unknown; + handleLLMNewToken?: ( + token: string, + idx: unknown, + runId: string, + parentRunId?: string, + tags?: string[], + fields?: unknown, + ) => Promise | unknown; + handleLLMEnd?: ( + output: unknown, + runId: string, + parentRunId?: string, + tags?: string[], + extraParams?: Record, + ) => Promise | unknown; + handleLLMError?: ( + error: Error, + runId: string, + parentRunId?: string, + tags?: string[], + extraParams?: Record, + ) => Promise | unknown; + handleChainStart?: ( + chain: { name?: string }, + inputs: Record, + runId: string, + parentRunId?: string, + tags?: string[], + metadata?: Record, + runType?: string, + runName?: string, + ) => Promise | unknown; + handleChainEnd?: ( + outputs: unknown, + runId: string, + parentRunId?: string, + tags?: string[], + kwargs?: { inputs?: Record }, + ) => Promise | unknown; + handleChainError?: ( + error: Error, + runId: string, + parentRunId?: string, + tags?: string[], + kwargs?: { inputs?: Record }, + ) => Promise | unknown; + handleToolStart?: ( + tool: { name?: string }, + input: string, + runId: string, + parentRunId?: string, + tags?: string[], + metadata?: Record, + runName?: string, + ) => Promise | unknown; + handleToolEnd?: (output: unknown, runId: string, parentRunId?: string, tags?: string[]) => Promise | unknown; + handleToolError?: (error: Error, runId: string, parentRunId?: string, tags?: string[]) => Promise | unknown; + + // LangChain class methods (required for BaseCallbackHandler compatibility) + copy(): unknown; + toJSON(): Record; + toJSONNotImplemented(): unknown; +} diff --git a/packages/core/src/utils/langchain/utils.ts b/packages/core/src/utils/langchain/utils.ts index 236216603e38..e4505fc94d43 100644 --- a/packages/core/src/utils/langchain/utils.ts +++ b/packages/core/src/utils/langchain/utils.ts @@ -24,7 +24,7 @@ import { GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE, } from '../ai/gen-ai-attributes'; import { LANGCHAIN_ORIGIN, ROLE_MAP } from './constants'; -import type { LangChainLLMResult, LangChainMessage, LangChainSerializedLLM } from './types'; +import type { LangChainLLMResult, LangChainMessage, LangChainSerialized } from './types'; /** * Assigns an attribute only when the value is neither `undefined` nor `null`. @@ -175,31 +175,32 @@ export function normalizeLangChainMessages(messages: LangChainMessage[]): Array< * Source precedence: * 1) `invocationParams` (highest) * 2) `langSmithMetadata` - * 3) `serialized.kwargs` * * Numeric values are set even when 0 (e.g. `temperature: 0`), but skipped if `NaN`. */ function extractCommonRequestAttributes( - serialized: LangChainSerializedLLM, + serialized: LangChainSerialized, invocationParams?: Record, langSmithMetadata?: Record, ): Record { const attrs: Record = {}; - const temperature = - invocationParams?.temperature ?? langSmithMetadata?.ls_temperature ?? serialized.kwargs?.temperature; + // Get kwargs if available (from constructor type) + const kwargs = 'kwargs' in serialized ? serialized.kwargs : undefined; + + const temperature = invocationParams?.temperature ?? langSmithMetadata?.ls_temperature ?? kwargs?.temperature; setNumberIfDefined(attrs, GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE, temperature); - const maxTokens = invocationParams?.max_tokens ?? langSmithMetadata?.ls_max_tokens ?? serialized.kwargs?.max_tokens; + const maxTokens = invocationParams?.max_tokens ?? langSmithMetadata?.ls_max_tokens ?? kwargs?.max_tokens; setNumberIfDefined(attrs, GEN_AI_REQUEST_MAX_TOKENS_ATTRIBUTE, maxTokens); - const topP = invocationParams?.top_p ?? serialized.kwargs?.top_p; + const topP = invocationParams?.top_p ?? kwargs?.top_p; setNumberIfDefined(attrs, GEN_AI_REQUEST_TOP_P_ATTRIBUTE, topP); - const frequencyPenalty = invocationParams?.frequency_penalty ?? serialized.kwargs?.frequency_penalty; + const frequencyPenalty = invocationParams?.frequency_penalty; setNumberIfDefined(attrs, GEN_AI_REQUEST_FREQUENCY_PENALTY_ATTRIBUTE, frequencyPenalty); - const presencePenalty = invocationParams?.presence_penalty ?? serialized.kwargs?.presence_penalty; + const presencePenalty = invocationParams?.presence_penalty; setNumberIfDefined(attrs, GEN_AI_REQUEST_PRESENCE_PENALTY_ATTRIBUTE, presencePenalty); // LangChain uses `stream`. We only set the attribute if the key actually exists @@ -218,7 +219,7 @@ function baseRequestAttributes( system: unknown, modelName: unknown, operation: 'pipeline' | 'chat', - serialized: LangChainSerializedLLM, + serialized: LangChainSerialized, invocationParams?: Record, langSmithMetadata?: Record, ): Record { @@ -239,7 +240,7 @@ function baseRequestAttributes( * messages to align with the chat schema used elsewhere. */ export function extractLLMRequestAttributes( - serialized: LangChainSerializedLLM, + llm: LangChainSerialized, prompts: string[], recordInputs: boolean, invocationParams?: Record, @@ -248,7 +249,7 @@ export function extractLLMRequestAttributes( const system = langSmithMetadata?.ls_provider; const modelName = invocationParams?.model ?? langSmithMetadata?.ls_model_name ?? 'unknown'; - const attrs = baseRequestAttributes(system, modelName, 'pipeline', serialized, invocationParams, langSmithMetadata); + const attrs = baseRequestAttributes(system, modelName, 'pipeline', llm, invocationParams, langSmithMetadata); if (recordInputs && Array.isArray(prompts) && prompts.length > 0) { const messages = prompts.map(p => ({ role: 'user', content: p })); @@ -267,19 +268,19 @@ export function extractLLMRequestAttributes( * - Provider system value falls back to `serialized.id?.[2]`. */ export function extractChatModelRequestAttributes( - serialized: LangChainSerializedLLM, - messages: LangChainMessage[][], + llm: LangChainSerialized, + langChainMessages: LangChainMessage[][], recordInputs: boolean, invocationParams?: Record, langSmithMetadata?: Record, ): Record { - const system = langSmithMetadata?.ls_provider ?? serialized.id?.[2]; + const system = langSmithMetadata?.ls_provider ?? llm.id?.[2]; const modelName = invocationParams?.model ?? langSmithMetadata?.ls_model_name ?? 'unknown'; - const attrs = baseRequestAttributes(system, modelName, 'chat', serialized, invocationParams, langSmithMetadata); + const attrs = baseRequestAttributes(system, modelName, 'chat', llm, invocationParams, langSmithMetadata); - if (recordInputs && Array.isArray(messages) && messages.length > 0) { - const normalized = normalizeLangChainMessages(messages.flat()); + if (recordInputs && Array.isArray(langChainMessages) && langChainMessages.length > 0) { + const normalized = normalizeLangChainMessages(langChainMessages.flat()); setIfDefined(attrs, GEN_AI_REQUEST_MESSAGES_ATTRIBUTE, asString(normalized)); } @@ -372,15 +373,16 @@ function addTokenUsageAttributes( * `stop_reason` (for providers that use it). */ export function extractLlmResponseAttributes( - response: LangChainLLMResult, + llmResult: LangChainLLMResult, recordOutputs: boolean, ): Record | undefined { - if (!response) return; + if (!llmResult) return; const attrs: Record = {}; - if (Array.isArray(response.generations)) { - const finishReasons = response.generations + if (Array.isArray(llmResult.generations)) { + const finishReasons = llmResult.generations + .flat() .map(g => g.generation_info?.finish_reason) .filter((r): r is string => typeof r === 'string'); @@ -389,10 +391,10 @@ export function extractLlmResponseAttributes( } // Tool calls metadata (names, IDs) are not PII, so capture them regardless of recordOutputs - addToolCallsAttributes(response.generations as LangChainMessage[][], attrs); + addToolCallsAttributes(llmResult.generations as LangChainMessage[][], attrs); if (recordOutputs) { - const texts = response.generations + const texts = llmResult.generations .flat() .map(gen => gen.text ?? gen.message?.content) .filter(t => typeof t === 'string'); @@ -403,9 +405,9 @@ export function extractLlmResponseAttributes( } } - addTokenUsageAttributes(response.llmOutput, attrs); + addTokenUsageAttributes(llmResult.llmOutput, attrs); - const llmOutput = response.llmOutput as { model_name?: string; model?: string; id?: string; stop_reason?: string }; + const llmOutput = llmResult.llmOutput as { model_name?: string; model?: string; id?: string; stop_reason?: string }; // Provider model identifier: `model_name` (OpenAI-style) or `model` (others) const modelName = llmOutput?.model_name ?? llmOutput?.model; if (modelName) setIfDefined(attrs, GEN_AI_RESPONSE_MODEL_ATTRIBUTE, modelName); diff --git a/packages/node/src/integrations/tracing/langchain/index.ts b/packages/node/src/integrations/tracing/langchain/index.ts index c88f85db926a..e575691b930f 100644 --- a/packages/node/src/integrations/tracing/langchain/index.ts +++ b/packages/node/src/integrations/tracing/langchain/index.ts @@ -47,10 +47,9 @@ const _langChainIntegration = ((options: LangChainOptions = {}) => { * @example * ```javascript * import * as Sentry from '@sentry/node'; - * import { createLangChainCallbackHandler } from '@sentry/core'; * import { ChatOpenAI } from '@langchain/openai'; * - * const sentryHandler = createLangChainCallbackHandler({ + * const sentryHandler = Sentry.createLangChainCallbackHandler({ * recordInputs: true, * recordOutputs: true * }); From 10380b46e1f0c4ba1a5f21c05904a9393d92f900 Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Tue, 21 Oct 2025 11:53:18 +0200 Subject: [PATCH 10/12] yarn lock fix --- yarn.lock | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/yarn.lock b/yarn.lock index 41f079f365fa..7cd8d25e9d52 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12543,7 +12543,7 @@ camelcase@5.0.0: resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.0.0.tgz#03295527d58bd3cd4aa75363f35b2e8d97be2f42" integrity sha512-faqwZqnWxbxn+F1d399ygeamQNy3lPp/H9H6rNrqYh4FSVCtcY+3cub1MxA8o9mDd55mM8Aghuu/kuyYA6VTsA== -camelcase@6, camelcase@^6.2.0, camelcase@^6.3.0: +camelcase@6, camelcase@^6.3.0: version "6.3.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a" integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== @@ -12553,10 +12553,6 @@ camelcase@^5.3.1: resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.3.1.tgz#e3c9b31569e106811df242f715725a1f4c494320" integrity sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg== -camelcase@^6.3.0: - version "6.3.0" - resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a" - integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== camelcase@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-7.0.1.tgz#f02e50af9fd7782bc8b88a3558c32fd3a388f048" From 00cdaf6efdebce771876e7f3eb272b5e8ee28d2f Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Wed, 22 Oct 2025 13:26:44 +0200 Subject: [PATCH 11/12] resolve comments --- .../suites/tracing/langchain/scenario.mjs | 2 +- packages/core/src/utils/langchain/index.ts | 333 ++++++++---------- packages/core/src/utils/langchain/utils.ts | 2 +- 3 files changed, 147 insertions(+), 190 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/scenario.mjs index 44b4175c5004..2c60e55ff77e 100644 --- a/dev-packages/node-integration-tests/suites/tracing/langchain/scenario.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/scenario.mjs @@ -97,7 +97,7 @@ async function run() { try { await errorModel.invoke('This will fail'); - } catch (error) { + } catch { // Expected error } }); diff --git a/packages/core/src/utils/langchain/index.ts b/packages/core/src/utils/langchain/index.ts index a34da24c197a..ef5f544a52f5 100644 --- a/packages/core/src/utils/langchain/index.ts +++ b/packages/core/src/utils/langchain/index.ts @@ -78,35 +78,31 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): metadata?: Record, _runName?: string, ) { - try { - const invocationParams = getInvocationParams(tags); - const attributes = extractLLMRequestAttributes( - llm as LangChainSerialized, - prompts, - recordInputs, - invocationParams, - metadata, - ); - const modelName = attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]; - const operationName = attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]; + const invocationParams = getInvocationParams(tags); + const attributes = extractLLMRequestAttributes( + llm as LangChainSerialized, + prompts, + recordInputs, + invocationParams, + metadata, + ); + const modelName = attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]; + const operationName = attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]; - startSpanManual( - { - name: `${operationName} ${modelName}`, - op: 'gen_ai.pipeline', - attributes: { - ...attributes, - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.pipeline', - }, + startSpanManual( + { + name: `${operationName} ${modelName}`, + op: 'gen_ai.pipeline', + attributes: { + ...attributes, + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.pipeline', }, - span => { - spanMap.set(runId, span); - return span; - }, - ); - } catch { - // Silently ignore errors, llm errors are captured by the handleLLMError handler - } + }, + span => { + spanMap.set(runId, span); + return span; + }, + ); }, // Chat Model Start Handler @@ -120,36 +116,31 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): metadata?: Record, _runName?: string, ) { - try { - const invocationParams = getInvocationParams(tags); - const attributes = extractChatModelRequestAttributes( - llm as LangChainSerialized, - messages as LangChainMessage[][], - recordInputs, - invocationParams, - metadata, - ); - const modelName = attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]; - const operationName = attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]; - - startSpanManual( - { - name: `${operationName} ${modelName}`, - op: 'gen_ai.chat', - attributes: { - ...attributes, - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', - }, - }, - span => { - spanMap.set(runId, span); + const invocationParams = getInvocationParams(tags); + const attributes = extractChatModelRequestAttributes( + llm as LangChainSerialized, + messages as LangChainMessage[][], + recordInputs, + invocationParams, + metadata, + ); + const modelName = attributes[GEN_AI_REQUEST_MODEL_ATTRIBUTE]; + const operationName = attributes[GEN_AI_OPERATION_NAME_ATTRIBUTE]; - return span; + startSpanManual( + { + name: `${operationName} ${modelName}`, + op: 'gen_ai.chat', + attributes: { + ...attributes, + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat', }, - ); - } catch { - // Silently ignore errors, chat model start errors are captured by the handleChatModelError handler - } + }, + span => { + spanMap.set(runId, span); + return span; + }, + ); }, // LLM End Handler - note: handleLLMEnd with capital LLM (used by both LLMs and chat models!) @@ -160,182 +151,148 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): _tags?: string[], _extraParams?: Record, ) { - try { - const span = spanMap.get(runId); - if (span) { - const attributes = extractLlmResponseAttributes(output as LangChainLLMResult, recordOutputs); - if (attributes) { - span.setAttributes(attributes); - } - exitSpan(runId); + const span = spanMap.get(runId); + if (span) { + const attributes = extractLlmResponseAttributes(output as LangChainLLMResult, recordOutputs); + if (attributes) { + span.setAttributes(attributes); } - } catch { - // Silently ignore errors, llm end errors are captured by the handleLLMError handler + exitSpan(runId); } }, // LLM Error Handler - note: handleLLMError with capital LLM handleLLMError(error: Error, runId: string) { - try { - const span = spanMap.get(runId); - if (span) { - span.setStatus({ code: SPAN_STATUS_ERROR, message: 'llm_error' }); - exitSpan(runId); - } - - captureException(error, { - mechanism: { - handled: false, - type: LANGCHAIN_ORIGIN, - data: { handler: 'handleLLMError' }, - }, - }); - } catch { - // silently ignore errors + const span = spanMap.get(runId); + if (span) { + span.setStatus({ code: SPAN_STATUS_ERROR, message: 'llm_error' }); + exitSpan(runId); } + + captureException(error, { + mechanism: { + handled: false, + type: `${LANGCHAIN_ORIGIN}.llm_error_handler`, + }, + }); }, // Chain Start Handler handleChainStart(chain: { name?: string }, inputs: Record, runId: string, _parentRunId?: string) { - try { - const chainName = chain.name || 'unknown_chain'; - const attributes: Record = { - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', - 'langchain.chain.name': chainName, - }; - - // Add inputs if recordInputs is enabled - if (recordInputs) { - attributes['langchain.chain.inputs'] = JSON.stringify(inputs); - } + const chainName = chain.name || 'unknown_chain'; + const attributes: Record = { + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain', + 'langchain.chain.name': chainName, + }; - startSpanManual( - { - name: `chain ${chainName}`, - op: 'gen_ai.invoke_agent', - attributes: { - ...attributes, - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', - }, - }, - span => { - spanMap.set(runId, span); + // Add inputs if recordInputs is enabled + if (recordInputs) { + attributes['langchain.chain.inputs'] = JSON.stringify(inputs); + } - return span; + startSpanManual( + { + name: `chain ${chainName}`, + op: 'gen_ai.invoke_agent', + attributes: { + ...attributes, + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', }, - ); - } catch { - // Silently ignore errors, chain start errors are captured by the handleChainError handler - } + }, + span => { + spanMap.set(runId, span); + return span; + }, + ); }, // Chain End Handler handleChainEnd(outputs: unknown, runId: string) { - try { - const span = spanMap.get(runId); - if (span) { - // Add outputs if recordOutputs is enabled - if (recordOutputs) { - span.setAttributes({ - 'langchain.chain.outputs': JSON.stringify(outputs), - }); - } - exitSpan(runId); + const span = spanMap.get(runId); + if (span) { + // Add outputs if recordOutputs is enabled + if (recordOutputs) { + span.setAttributes({ + 'langchain.chain.outputs': JSON.stringify(outputs), + }); } - } catch { - // Silently ignore errors, chain end errors are captured by the handleChainError handler + exitSpan(runId); } }, // Chain Error Handler handleChainError(error: Error, runId: string) { - try { - const span = spanMap.get(runId); - if (span) { - span.setStatus({ code: SPAN_STATUS_ERROR, message: 'chain_error' }); - exitSpan(runId); - } - - captureException(error, { - mechanism: { - handled: false, - type: LANGCHAIN_ORIGIN, - }, - }); - } catch { - // silently ignore errors + const span = spanMap.get(runId); + if (span) { + span.setStatus({ code: SPAN_STATUS_ERROR, message: 'chain_error' }); + exitSpan(runId); } + + captureException(error, { + mechanism: { + handled: false, + type: `${LANGCHAIN_ORIGIN}.chain_error_handler`, + }, + }); }, // Tool Start Handler handleToolStart(tool: { name?: string }, input: string, runId: string, _parentRunId?: string) { - try { - const toolName = tool.name || 'unknown_tool'; - const attributes: Record = { - [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: LANGCHAIN_ORIGIN, - 'gen_ai.tool.name': toolName, - }; + const toolName = tool.name || 'unknown_tool'; + const attributes: Record = { + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: LANGCHAIN_ORIGIN, + 'gen_ai.tool.name': toolName, + }; - // Add input if recordInputs is enabled - if (recordInputs) { - attributes['gen_ai.tool.input'] = input; - } + // Add input if recordInputs is enabled + if (recordInputs) { + attributes['gen_ai.tool.input'] = input; + } - startSpanManual( - { - name: `execute_tool ${toolName}`, - op: 'gen_ai.execute_tool', - attributes: { - ...attributes, - [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.execute_tool', - }, - }, - span => { - spanMap.set(runId, span); - return span; + startSpanManual( + { + name: `execute_tool ${toolName}`, + op: 'gen_ai.execute_tool', + attributes: { + ...attributes, + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.execute_tool', }, - ); - } catch { - // Silently ignore errors, tool start errors are captured by the handleToolError handler - } + }, + span => { + spanMap.set(runId, span); + return span; + }, + ); }, // Tool End Handler handleToolEnd(output: unknown, runId: string) { - try { - const span = spanMap.get(runId); - if (span) { - // Add output if recordOutputs is enabled - if (recordOutputs) { - span.setAttributes({ - 'gen_ai.tool.output': JSON.stringify(output), - }); - } - exitSpan(runId); + const span = spanMap.get(runId); + if (span) { + // Add output if recordOutputs is enabled + if (recordOutputs) { + span.setAttributes({ + 'gen_ai.tool.output': JSON.stringify(output), + }); } - } catch { - // Silently ignore errors, tool start errors are captured by the handleToolError handler + exitSpan(runId); } }, // Tool Error Handler handleToolError(error: Error, runId: string) { - try { - const span = spanMap.get(runId); - if (span) { - span.setStatus({ code: SPAN_STATUS_ERROR, message: 'tool_error' }); - exitSpan(runId); - } - - captureException(error, { - mechanism: { - handled: false, - type: LANGCHAIN_ORIGIN, - }, - }); - } catch { - // silently ignore errors + const span = spanMap.get(runId); + if (span) { + span.setStatus({ code: SPAN_STATUS_ERROR, message: 'tool_error' }); + exitSpan(runId); } + + captureException(error, { + mechanism: { + handled: false, + type: `${LANGCHAIN_ORIGIN}.tool_error_handler`, + }, + }); }, // LangChain BaseCallbackHandler required methods diff --git a/packages/core/src/utils/langchain/utils.ts b/packages/core/src/utils/langchain/utils.ts index e4505fc94d43..8464e71aecb0 100644 --- a/packages/core/src/utils/langchain/utils.ts +++ b/packages/core/src/utils/langchain/utils.ts @@ -33,7 +33,7 @@ import type { LangChainLLMResult, LangChainMessage, LangChainSerialized } from ' * It also preserves falsy-but-valid values like `0` and `""`. */ const setIfDefined = (target: Record, key: string, value: unknown): void => { - if (value !== undefined && value !== null) target[key] = value as SpanAttributeValue; + if (value != null) target[key] = value as SpanAttributeValue; }; /** From 6cda4e0355eeeca2143137f6eeaa0bd0d35c8611 Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Wed, 22 Oct 2025 14:37:07 +0200 Subject: [PATCH 12/12] check if span is recording before adding to it --- packages/core/src/utils/langchain/index.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/core/src/utils/langchain/index.ts b/packages/core/src/utils/langchain/index.ts index ef5f544a52f5..1930be794be5 100644 --- a/packages/core/src/utils/langchain/index.ts +++ b/packages/core/src/utils/langchain/index.ts @@ -37,7 +37,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): */ const exitSpan = (runId: string): void => { const span = spanMap.get(runId); - if (span) { + if (span?.isRecording()) { span.end(); spanMap.delete(runId); } @@ -152,7 +152,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): _extraParams?: Record, ) { const span = spanMap.get(runId); - if (span) { + if (span?.isRecording()) { const attributes = extractLlmResponseAttributes(output as LangChainLLMResult, recordOutputs); if (attributes) { span.setAttributes(attributes); @@ -164,7 +164,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): // LLM Error Handler - note: handleLLMError with capital LLM handleLLMError(error: Error, runId: string) { const span = spanMap.get(runId); - if (span) { + if (span?.isRecording()) { span.setStatus({ code: SPAN_STATUS_ERROR, message: 'llm_error' }); exitSpan(runId); } @@ -209,7 +209,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): // Chain End Handler handleChainEnd(outputs: unknown, runId: string) { const span = spanMap.get(runId); - if (span) { + if (span?.isRecording()) { // Add outputs if recordOutputs is enabled if (recordOutputs) { span.setAttributes({ @@ -223,7 +223,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): // Chain Error Handler handleChainError(error: Error, runId: string) { const span = spanMap.get(runId); - if (span) { + if (span?.isRecording()) { span.setStatus({ code: SPAN_STATUS_ERROR, message: 'chain_error' }); exitSpan(runId); } @@ -268,7 +268,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): // Tool End Handler handleToolEnd(output: unknown, runId: string) { const span = spanMap.get(runId); - if (span) { + if (span?.isRecording()) { // Add output if recordOutputs is enabled if (recordOutputs) { span.setAttributes({ @@ -282,7 +282,7 @@ export function createLangChainCallbackHandler(options: LangChainOptions = {}): // Tool Error Handler handleToolError(error: Error, runId: string) { const span = spanMap.get(runId); - if (span) { + if (span?.isRecording()) { span.setStatus({ code: SPAN_STATUS_ERROR, message: 'tool_error' }); exitSpan(runId); }