diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument-with-pii.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument-with-pii.mjs new file mode 100644 index 000000000000..3d911666a7d7 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument-with-pii.mjs @@ -0,0 +1,17 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + sendDefaultPii: true, + transport: loggingTransport, + beforeSendTransaction: event => { + // Filter out mock express server transactions + if (event.transaction.includes('/v1/messages') || event.transaction.includes('/v1/chat/completions')) { + return null; + } + return event; + }, +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument.mjs new file mode 100644 index 000000000000..05985d888de9 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/instrument.mjs @@ -0,0 +1,17 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + sendDefaultPii: false, + transport: loggingTransport, + beforeSendTransaction: event => { + // Filter out mock express server transactions + if (event.transaction.includes('/v1/messages') || event.transaction.includes('/v1/chat/completions')) { + return null; + } + return event; + }, +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-init-chat-model.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-init-chat-model.mjs new file mode 100644 index 000000000000..1ca0337f238f --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-init-chat-model.mjs @@ -0,0 +1,110 @@ +import * as Sentry from '@sentry/node'; +import express from 'express'; +import { initChatModel } from 'langchain'; + +function startMockOpenAIServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1/chat/completions', (req, res) => { + const model = req.body.model; + + if (model === 'error-model') { + res.status(404).json({ + error: { + message: 'Model not found', + type: 'invalid_request_error', + param: null, + code: 'model_not_found', + }, + }); + return; + } + + // Simulate OpenAI response + res.json({ + id: 'chatcmpl-init-test-123', + object: 'chat.completion', + created: 1677652288, + model: model, + system_fingerprint: 'fp_44709d6fcb', + choices: [ + { + index: 0, + message: { + role: 'assistant', + content: 'Hello from initChatModel!', + }, + finish_reason: 'stop', + }, + ], + usage: { + prompt_tokens: 8, + completion_tokens: 12, + total_tokens: 20, + }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockOpenAIServer(); + const baseUrl = `http://localhost:${server.address().port}/v1`; + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + // Set OpenAI API key in environment + process.env.OPENAI_API_KEY = 'mock-api-key'; + + // Test 1: Initialize chat model using unified API with model string + const model1 = await initChatModel('gpt-4o', { + temperature: 0.7, + maxTokens: 100, + modelProvider: 'openai', + configurableFields: ['model'], + configuration: { + baseURL: baseUrl, + }, + }); + + await model1.invoke('Tell me about LangChain'); + + // Test 2: Initialize with different model + const model2 = await initChatModel('gpt-3.5-turbo', { + temperature: 0.5, + modelProvider: 'openai', + configuration: { + baseURL: baseUrl, + }, + }); + + await model2.invoke([ + { role: 'system', content: 'You are a helpful assistant' }, + { role: 'user', content: 'What is AI?' }, + ]); + + // Test 3: Error handling + try { + const errorModel = await initChatModel('error-model', { + modelProvider: 'openai', + configuration: { + baseURL: baseUrl, + }, + }); + await errorModel.invoke('This will fail'); + } catch { + // Expected error + } + }); + + await Sentry.flush(2000); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-message-truncation.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-message-truncation.mjs new file mode 100644 index 000000000000..6dafe8572cec --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-message-truncation.mjs @@ -0,0 +1,72 @@ +import { ChatAnthropic } from '@langchain/anthropic'; +import * as Sentry from '@sentry/node'; +import express from 'express'; + +function startMockAnthropicServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1/messages', (req, res) => { + const model = req.body.model; + + res.json({ + id: 'msg_truncation_test', + type: 'message', + role: 'assistant', + content: [ + { + type: 'text', + text: 'Response to truncated messages', + }, + ], + model: model, + stop_reason: 'end_turn', + stop_sequence: null, + usage: { + input_tokens: 10, + output_tokens: 15, + }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockAnthropicServer(); + const baseUrl = `http://localhost:${server.address().port}`; + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const model = new ChatAnthropic({ + model: 'claude-3-5-sonnet-20241022', + apiKey: 'mock-api-key', + clientOptions: { + baseURL: baseUrl, + }, + }); + + const largeContent1 = 'A'.repeat(15000); // ~15KB + const largeContent2 = 'B'.repeat(15000); // ~15KB + const largeContent3 = 'C'.repeat(25000); // ~25KB (will be truncated) + + // Create one very large string that gets truncated to only include Cs + await model.invoke(largeContent3 + largeContent2); + + // Create an array of messages that gets truncated to only include the last message (result should again contain only Cs) + await model.invoke([ + { role: 'system', content: largeContent1 }, + { role: 'user', content: largeContent2 }, + { role: 'user', content: largeContent3 }, + ]); + }); + + await Sentry.flush(2000); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-openai-before-langchain.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-openai-before-langchain.mjs new file mode 100644 index 000000000000..f194acb1672b --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-openai-before-langchain.mjs @@ -0,0 +1,95 @@ +import * as Sentry from '@sentry/node'; +import express from 'express'; + +function startMockAnthropicServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1/messages', (req, res) => { + res.json({ + id: 'msg_test123', + type: 'message', + role: 'assistant', + content: [ + { + type: 'text', + text: 'Mock response from Anthropic!', + }, + ], + model: req.body.model, + stop_reason: 'end_turn', + stop_sequence: null, + usage: { + input_tokens: 10, + output_tokens: 15, + }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockAnthropicServer(); + const baseURL = `http://localhost:${server.address().port}`; + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + // EDGE CASE: Import and instantiate Anthropic client BEFORE LangChain is imported + // This simulates the timing issue where a user creates an Anthropic client in one file + // before importing LangChain in another file + const { default: Anthropic } = await import('@anthropic-ai/sdk'); + const anthropicClient = new Anthropic({ + apiKey: 'mock-api-key', + baseURL, + }); + + // Use the Anthropic client directly - this will be instrumented by the Anthropic integration + await anthropicClient.messages.create({ + model: 'claude-3-5-sonnet-20241022', + messages: [{ role: 'user', content: 'Direct Anthropic call' }], + temperature: 0.7, + max_tokens: 100, + }); + + // NOW import LangChain - at this point it will mark Anthropic to be skipped + // But the client created above is already instrumented + const { ChatAnthropic } = await import('@langchain/anthropic'); + + // Create a LangChain model - this uses Anthropic under the hood + const langchainModel = new ChatAnthropic({ + model: 'claude-3-5-sonnet-20241022', + temperature: 0.7, + maxTokens: 100, + apiKey: 'mock-api-key', + clientOptions: { + baseURL, + }, + }); + + // Use LangChain - this will be instrumented by LangChain integration + await langchainModel.invoke('LangChain Anthropic call'); + + // Create ANOTHER Anthropic client after LangChain was imported + // This one should NOT be instrumented (skip mechanism works correctly) + const anthropicClient2 = new Anthropic({ + apiKey: 'mock-api-key', + baseURL, + }); + + await anthropicClient2.messages.create({ + model: 'claude-3-5-sonnet-20241022', + messages: [{ role: 'user', content: 'Second direct Anthropic call' }], + temperature: 0.7, + max_tokens: 100, + }); + }); + + await Sentry.flush(2000); + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-tools.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-tools.mjs new file mode 100644 index 000000000000..256ee4568884 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario-tools.mjs @@ -0,0 +1,90 @@ +import { ChatAnthropic } from '@langchain/anthropic'; +import * as Sentry from '@sentry/node'; +import express from 'express'; + +function startMockAnthropicServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1/messages', (req, res) => { + const model = req.body.model; + + // Simulate tool call response + res.json({ + id: 'msg_tool_test_123', + type: 'message', + role: 'assistant', + model: model, + content: [ + { + type: 'text', + text: 'Let me check the weather for you.', + }, + { + type: 'tool_use', + id: 'toolu_01A09q90qw90lq917835lq9', + name: 'get_weather', + input: { location: 'San Francisco, CA' }, + }, + { + type: 'text', + text: 'The weather looks great!', + }, + ], + stop_reason: 'tool_use', + stop_sequence: null, + usage: { + input_tokens: 20, + output_tokens: 30, + }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockAnthropicServer(); + const baseUrl = `http://localhost:${server.address().port}`; + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const model = new ChatAnthropic({ + model: 'claude-3-5-sonnet-20241022', + temperature: 0.7, + maxTokens: 150, + apiKey: 'mock-api-key', + clientOptions: { + baseURL: baseUrl, + }, + }); + + await model.invoke('What is the weather in San Francisco?', { + tools: [ + { + name: 'get_weather', + description: 'Get the current weather in a given location', + input_schema: { + type: 'object', + properties: { + location: { + type: 'string', + description: 'The city and state, e.g. San Francisco, CA', + }, + }, + required: ['location'], + }, + }, + ], + }); + }); + + await Sentry.flush(2000); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario.mjs new file mode 100644 index 000000000000..2c60e55ff77e --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/scenario.mjs @@ -0,0 +1,110 @@ +import { ChatAnthropic } from '@langchain/anthropic'; +import * as Sentry from '@sentry/node'; +import express from 'express'; + +function startMockAnthropicServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1/messages', (req, res) => { + const model = req.body.model; + + if (model === 'error-model') { + res + .status(400) + .set('request-id', 'mock-request-123') + .json({ + type: 'error', + error: { + type: 'invalid_request_error', + message: 'Model not found', + }, + }); + return; + } + + // Simulate basic response + res.json({ + id: 'msg_test123', + type: 'message', + role: 'assistant', + content: [ + { + type: 'text', + text: 'Mock response from Anthropic!', + }, + ], + model: model, + stop_reason: 'end_turn', + stop_sequence: null, + usage: { + input_tokens: 10, + output_tokens: 15, + }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockAnthropicServer(); + const baseUrl = `http://localhost:${server.address().port}`; + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + // Test 1: Basic chat model invocation + const model1 = new ChatAnthropic({ + model: 'claude-3-5-sonnet-20241022', + temperature: 0.7, + maxTokens: 100, + apiKey: 'mock-api-key', + clientOptions: { + baseURL: baseUrl, + }, + }); + + await model1.invoke('Tell me a joke'); + + // Test 2: Chat with different model + const model2 = new ChatAnthropic({ + model: 'claude-3-opus-20240229', + temperature: 0.9, + topP: 0.95, + maxTokens: 200, + apiKey: 'mock-api-key', + clientOptions: { + baseURL: baseUrl, + }, + }); + + await model2.invoke([ + { role: 'system', content: 'You are a helpful assistant' }, + { role: 'user', content: 'What is the capital of France?' }, + ]); + + // Test 3: Error handling + const errorModel = new ChatAnthropic({ + model: 'error-model', + apiKey: 'mock-api-key', + clientOptions: { + baseURL: baseUrl, + }, + }); + + try { + await errorModel.invoke('This will fail'); + } catch { + // Expected error + } + }); + + await Sentry.flush(2000); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts new file mode 100644 index 000000000000..eaa40887fc66 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts @@ -0,0 +1,449 @@ +import { afterAll, expect } from 'vitest'; +import { conditionalTest } from '../../../../utils'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../../utils/runner'; + +// LangChain v1 requires Node.js 20+ (dropped Node 18 support) +// See: https://docs.langchain.com/oss/javascript/migrate/langgraph-v1#dropped-node-18-support +conditionalTest({ min: 20 })('LangChain integration (v1)', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + const EXPECTED_TRANSACTION_DEFAULT_PII_FALSE = { + transaction: 'main', + spans: expect.arrayContaining([ + // First span - chat model with claude-3-5-sonnet + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'auto.ai.langchain', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-3-5-sonnet-20241022', + 'gen_ai.request.temperature': 0.7, + 'gen_ai.request.max_tokens': 100, + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 15, + 'gen_ai.usage.total_tokens': 25, + 'gen_ai.response.id': expect.any(String), + 'gen_ai.response.model': expect.any(String), + 'gen_ai.response.stop_reason': expect.any(String), + }), + description: 'chat claude-3-5-sonnet-20241022', + op: 'gen_ai.chat', + origin: 'auto.ai.langchain', + status: 'ok', + }), + // Second span - chat model with claude-3-opus + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'auto.ai.langchain', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-3-opus-20240229', + 'gen_ai.request.temperature': 0.9, + 'gen_ai.request.top_p': 0.95, + 'gen_ai.request.max_tokens': 200, + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 15, + 'gen_ai.usage.total_tokens': 25, + 'gen_ai.response.id': expect.any(String), + 'gen_ai.response.model': expect.any(String), + 'gen_ai.response.stop_reason': expect.any(String), + }), + description: 'chat claude-3-opus-20240229', + op: 'gen_ai.chat', + origin: 'auto.ai.langchain', + status: 'ok', + }), + // Third span - error handling + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'auto.ai.langchain', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'error-model', + }), + description: 'chat error-model', + op: 'gen_ai.chat', + origin: 'auto.ai.langchain', + status: 'internal_error', + }), + ]), + }; + + const EXPECTED_TRANSACTION_DEFAULT_PII_TRUE = { + transaction: 'main', + spans: expect.arrayContaining([ + // First span - chat model with PII + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'auto.ai.langchain', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-3-5-sonnet-20241022', + 'gen_ai.request.temperature': 0.7, + 'gen_ai.request.max_tokens': 100, + 'gen_ai.request.messages': expect.any(String), // Should include messages when recordInputs: true + 'gen_ai.response.text': expect.any(String), // Should include response when recordOutputs: true + 'gen_ai.response.id': expect.any(String), + 'gen_ai.response.model': expect.any(String), + 'gen_ai.response.stop_reason': expect.any(String), + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 15, + 'gen_ai.usage.total_tokens': 25, + }), + description: 'chat claude-3-5-sonnet-20241022', + op: 'gen_ai.chat', + origin: 'auto.ai.langchain', + status: 'ok', + }), + // Second span - chat model with PII + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'auto.ai.langchain', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-3-opus-20240229', + 'gen_ai.request.temperature': 0.9, + 'gen_ai.request.top_p': 0.95, + 'gen_ai.request.max_tokens': 200, + 'gen_ai.request.messages': expect.any(String), // Should include messages when recordInputs: true + 'gen_ai.response.text': expect.any(String), // Should include response when recordOutputs: true + 'gen_ai.response.id': expect.any(String), + 'gen_ai.response.model': expect.any(String), + 'gen_ai.response.stop_reason': expect.any(String), + 'gen_ai.usage.input_tokens': 10, + 'gen_ai.usage.output_tokens': 15, + 'gen_ai.usage.total_tokens': 25, + }), + description: 'chat claude-3-opus-20240229', + op: 'gen_ai.chat', + origin: 'auto.ai.langchain', + status: 'ok', + }), + // Third span - error handling with PII + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'auto.ai.langchain', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'error-model', + 'gen_ai.request.messages': expect.any(String), // Should include messages when recordInputs: true + }), + description: 'chat error-model', + op: 'gen_ai.chat', + origin: 'auto.ai.langchain', + status: 'internal_error', + }), + ]), + }; + + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('creates langchain related spans with sendDefaultPii: false', async () => { + await createRunner() + .ignore('event') + .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_FALSE }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + langchain: '^1.0.0', + '@langchain/core': '^1.0.0', + '@langchain/anthropic': '^1.0.0', + }, + }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + test('creates langchain related spans with sendDefaultPii: true', async () => { + await createRunner() + .ignore('event') + .expect({ transaction: EXPECTED_TRANSACTION_DEFAULT_PII_TRUE }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + langchain: '^1.0.0', + '@langchain/core': '^1.0.0', + '@langchain/anthropic': '^1.0.0', + }, + }, + ); + + const EXPECTED_TRANSACTION_TOOL_CALLS = { + transaction: 'main', + spans: expect.arrayContaining([ + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'auto.ai.langchain', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-3-5-sonnet-20241022', + 'gen_ai.request.temperature': 0.7, + 'gen_ai.request.max_tokens': 150, + 'gen_ai.usage.input_tokens': 20, + 'gen_ai.usage.output_tokens': 30, + 'gen_ai.usage.total_tokens': 50, + 'gen_ai.response.id': expect.any(String), + 'gen_ai.response.model': expect.any(String), + 'gen_ai.response.stop_reason': 'tool_use', + 'gen_ai.response.tool_calls': expect.any(String), + }), + description: 'chat claude-3-5-sonnet-20241022', + op: 'gen_ai.chat', + origin: 'auto.ai.langchain', + status: 'ok', + }), + ]), + }; + + createEsmAndCjsTests( + __dirname, + 'scenario-tools.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('creates langchain spans with tool calls', async () => { + await createRunner() + .ignore('event') + .expect({ transaction: EXPECTED_TRANSACTION_TOOL_CALLS }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + langchain: '^1.0.0', + '@langchain/core': '^1.0.0', + '@langchain/anthropic': '^1.0.0', + }, + }, + ); + + const EXPECTED_TRANSACTION_MESSAGE_TRUNCATION = { + transaction: 'main', + spans: expect.arrayContaining([ + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'auto.ai.langchain', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-3-5-sonnet-20241022', + // Messages should be present and should include truncated string input (contains only Cs) + 'gen_ai.request.messages': expect.stringMatching(/^\[\{"role":"user","content":"C+"\}\]$/), + }), + description: 'chat claude-3-5-sonnet-20241022', + op: 'gen_ai.chat', + origin: 'auto.ai.langchain', + status: 'ok', + }), + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'auto.ai.langchain', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-3-5-sonnet-20241022', + // Messages should be present (truncation happened) and should be a JSON array of a single index (contains only Cs) + 'gen_ai.request.messages': expect.stringMatching(/^\[\{"role":"user","content":"C+"\}\]$/), + }), + description: 'chat claude-3-5-sonnet-20241022', + op: 'gen_ai.chat', + origin: 'auto.ai.langchain', + status: 'ok', + }), + ]), + }; + + createEsmAndCjsTests( + __dirname, + 'scenario-message-truncation.mjs', + 'instrument-with-pii.mjs', + (createRunner, test) => { + test('truncates messages when they exceed byte limit', async () => { + await createRunner() + .ignore('event') + .expect({ transaction: EXPECTED_TRANSACTION_MESSAGE_TRUNCATION }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + langchain: '^1.0.0', + '@langchain/core': '^1.0.0', + '@langchain/anthropic': '^1.0.0', + }, + }, + ); + + createEsmAndCjsTests( + __dirname, + 'scenario-openai-before-langchain.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('demonstrates timing issue with duplicate spans (ESM only)', async () => { + await createRunner() + .ignore('event') + .expect({ + transaction: event => { + // This test highlights the limitation: if a user creates an Anthropic client + // before importing LangChain, that client will still be instrumented and + // could cause duplicate spans when used alongside LangChain. + + const spans = event.spans || []; + + // First call: Direct Anthropic call made BEFORE LangChain import + // This should have Anthropic instrumentation (origin: 'auto.ai.anthropic') + const firstAnthropicSpan = spans.find( + span => + span.description === 'messages claude-3-5-sonnet-20241022' && span.origin === 'auto.ai.anthropic', + ); + + // Second call: LangChain call + // This should have LangChain instrumentation (origin: 'auto.ai.langchain') + const langchainSpan = spans.find( + span => span.description === 'chat claude-3-5-sonnet-20241022' && span.origin === 'auto.ai.langchain', + ); + + // Third call: Direct Anthropic call made AFTER LangChain import + // This should NOT have Anthropic instrumentation (skip works correctly) + // Count how many Anthropic spans we have - should be exactly 1 + const anthropicSpans = spans.filter( + span => + span.description === 'messages claude-3-5-sonnet-20241022' && span.origin === 'auto.ai.anthropic', + ); + + // Verify the edge case limitation: + // - First Anthropic client (created before LangChain) IS instrumented + expect(firstAnthropicSpan).toBeDefined(); + expect(firstAnthropicSpan?.origin).toBe('auto.ai.anthropic'); + + // - LangChain call IS instrumented by LangChain + expect(langchainSpan).toBeDefined(); + expect(langchainSpan?.origin).toBe('auto.ai.langchain'); + + // - Second Anthropic client (created after LangChain) is NOT instrumented + // This demonstrates that the skip mechanism works for NEW clients + // We should only have ONE Anthropic span (the first one), not two + expect(anthropicSpans).toHaveLength(1); + }, + }) + .start() + .completed(); + }); + }, + { + failsOnCjs: true, + additionalDependencies: { + langchain: '^1.0.0', + '@langchain/core': '^1.0.0', + '@langchain/anthropic': '^1.0.0', + }, + }, + ); + + const EXPECTED_TRANSACTION_INIT_CHAT_MODEL = { + transaction: 'main', + spans: expect.arrayContaining([ + // First span - initChatModel with gpt-4o + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'auto.ai.langchain', + 'gen_ai.system': 'openai', + 'gen_ai.request.model': 'gpt-4o', + 'gen_ai.request.temperature': 0.7, + 'gen_ai.request.max_tokens': 100, + 'gen_ai.usage.input_tokens': 8, + 'gen_ai.usage.output_tokens': 12, + 'gen_ai.usage.total_tokens': 20, + 'gen_ai.response.id': expect.any(String), + 'gen_ai.response.model': 'gpt-4o', + 'gen_ai.response.stop_reason': 'stop', + }), + description: 'chat gpt-4o', + op: 'gen_ai.chat', + origin: 'auto.ai.langchain', + status: 'ok', + }), + // Second span - initChatModel with gpt-3.5-turbo + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'auto.ai.langchain', + 'gen_ai.system': 'openai', + 'gen_ai.request.model': 'gpt-3.5-turbo', + 'gen_ai.request.temperature': 0.5, + 'gen_ai.usage.input_tokens': 8, + 'gen_ai.usage.output_tokens': 12, + 'gen_ai.usage.total_tokens': 20, + 'gen_ai.response.id': expect.any(String), + 'gen_ai.response.model': 'gpt-3.5-turbo', + 'gen_ai.response.stop_reason': 'stop', + }), + description: 'chat gpt-3.5-turbo', + op: 'gen_ai.chat', + origin: 'auto.ai.langchain', + status: 'ok', + }), + // Third span - error handling + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'chat', + 'sentry.op': 'gen_ai.chat', + 'sentry.origin': 'auto.ai.langchain', + 'gen_ai.system': 'openai', + 'gen_ai.request.model': 'error-model', + }), + description: 'chat error-model', + op: 'gen_ai.chat', + origin: 'auto.ai.langchain', + status: 'internal_error', + }), + ]), + }; + + createEsmAndCjsTests( + __dirname, + 'scenario-init-chat-model.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('creates langchain spans using initChatModel with OpenAI', async () => { + await createRunner() + .ignore('event') + .expect({ transaction: EXPECTED_TRANSACTION_INIT_CHAT_MODEL }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + langchain: '^1.0.0', + '@langchain/core': '^1.0.0', + '@langchain/openai': '^1.0.0', + }, + }, + ); +}); diff --git a/packages/core/src/tracing/langchain/types.ts b/packages/core/src/tracing/langchain/types.ts index e08542eefd60..7379de764817 100644 --- a/packages/core/src/tracing/langchain/types.ts +++ b/packages/core/src/tracing/langchain/types.ts @@ -46,7 +46,11 @@ export interface LangChainMessage { additional_kwargs?: Record; // LangChain serialized format lc?: number; - id?: string[]; + id?: string[] | string; + response_metadata?: { + model_name?: string; + finish_reason?: string; + }; kwargs?: { [key: string]: unknown; content?: string; @@ -70,6 +74,12 @@ export interface LangChainLLMResult { finish_reason?: string; logprobs?: unknown; }; + // v1+ uses generationInfo instead of generation_info + generationInfo?: { + [key: string]: unknown; + finish_reason?: string; + logprobs?: unknown; + }; }> >; llmOutput?: { diff --git a/packages/core/src/tracing/langchain/utils.ts b/packages/core/src/tracing/langchain/utils.ts index caacf5059bdc..9a8fa9aed26d 100644 --- a/packages/core/src/tracing/langchain/utils.ts +++ b/packages/core/src/tracing/langchain/utils.ts @@ -385,7 +385,17 @@ export function extractLlmResponseAttributes( if (Array.isArray(llmResult.generations)) { const finishReasons = llmResult.generations .flat() - .map(g => g.generation_info?.finish_reason) + .map(g => { + // v1 uses generationInfo.finish_reason + if (g.generationInfo?.finish_reason) { + return g.generationInfo.finish_reason; + } + // v0.3+ uses generation_info.finish_reason + if (g.generation_info?.finish_reason) { + return g.generation_info.finish_reason; + } + return null; + }) .filter((r): r is string => typeof r === 'string'); if (finishReasons.length > 0) { @@ -409,17 +419,27 @@ export function extractLlmResponseAttributes( addTokenUsageAttributes(llmResult.llmOutput, attrs); - const llmOutput = llmResult.llmOutput as { model_name?: string; model?: string; id?: string; stop_reason?: string }; + const llmOutput = llmResult.llmOutput; + + // Extract from v1 generations structure if available + const firstGeneration = llmResult.generations?.[0]?.[0]; + const v1Message = firstGeneration?.message; + // Provider model identifier: `model_name` (OpenAI-style) or `model` (others) - const modelName = llmOutput?.model_name ?? llmOutput?.model; + // v1 stores this in message.response_metadata.model_name + const modelName = llmOutput?.model_name ?? llmOutput?.model ?? v1Message?.response_metadata?.model_name; if (modelName) setIfDefined(attrs, GEN_AI_RESPONSE_MODEL_ATTRIBUTE, modelName); - if (llmOutput?.id) { - setIfDefined(attrs, GEN_AI_RESPONSE_ID_ATTRIBUTE, llmOutput.id); + // Response ID: v1 stores this in message.id + const responseId = llmOutput?.id ?? v1Message?.id; + if (responseId) { + setIfDefined(attrs, GEN_AI_RESPONSE_ID_ATTRIBUTE, responseId); } - if (llmOutput?.stop_reason) { - setIfDefined(attrs, GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE, asString(llmOutput.stop_reason)); + // Stop reason: v1 stores this in message.response_metadata.finish_reason + const stopReason = llmOutput?.stop_reason ?? v1Message?.response_metadata?.finish_reason; + if (stopReason) { + setIfDefined(attrs, GEN_AI_RESPONSE_STOP_REASON_ATTRIBUTE, asString(stopReason)); } return attrs; diff --git a/packages/node/src/integrations/tracing/langchain/instrumentation.ts b/packages/node/src/integrations/tracing/langchain/instrumentation.ts index 09c6002a4629..8d5c42111f3d 100644 --- a/packages/node/src/integrations/tracing/langchain/instrumentation.ts +++ b/packages/node/src/integrations/tracing/langchain/instrumentation.ts @@ -16,7 +16,7 @@ import { SDK_VERSION, } from '@sentry/core'; -const supportedVersions = ['>=0.1.0 <1.0.0']; +const supportedVersions = ['>=0.1.0 <2.0.0']; type LangChainInstrumentationOptions = InstrumentationConfig & LangChainOptions; @@ -143,6 +143,25 @@ export class SentryLangChainInstrumentation extends InstrumentationBase exports, + [ + // To catch the CJS build that contains ConfigurableModel / initChatModel for v1 + new InstrumentationNodeModuleFile( + 'langchain/dist/chat_models/universal.cjs', + supportedVersions, + this._patch.bind(this), + exports => exports, + ), + ], + ), + ); + return modules; } @@ -193,14 +212,13 @@ export class SentryLangChainInstrumentation extends InstrumentationBase { - if (typeof exp !== 'function') { - return false; - } - return knownChatModelNames.includes(exp.name); + const exportsToPatch = (exports.universal_exports ?? exports) as Record; + + const chatModelClass = Object.values(exportsToPatch).find(exp => { + return typeof exp === 'function' && knownChatModelNames.includes(exp.name); }) as { prototype: unknown; name: string } | undefined; if (!chatModelClass) {