From 37b53648fe12a60547e28e61b9a544edde12e1bf Mon Sep 17 00:00:00 2001 From: isaacs Date: Thu, 4 Dec 2025 14:06:53 -0800 Subject: [PATCH] feat(tracing): strip inline media from messages (#18413) This is the functional portion addressing JS-1002. Prior to truncating text messages for their byte length, any inline base64-encoded media properties are filtered out. This allows the message to possibly be included in the span, indicating to the user that a media object was present, without overflowing the allotted buffer for sending data. If a media message is not removed, the fallback is still to simply remove it if its overhead grows too large. Re JS-1002 Re GH-17810 --- .size-limit.js | 2 +- .../anthropic/scenario-media-truncation.mjs | 79 ++++ .../suites/tracing/anthropic/test.ts | 48 +++ .../core/src/tracing/ai/messageTruncation.ts | 184 +++++++++- .../lib/tracing/ai-message-truncation.test.ts | 336 ++++++++++++++++++ 5 files changed, 633 insertions(+), 16 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-media-truncation.mjs create mode 100644 packages/core/test/lib/tracing/ai-message-truncation.test.ts diff --git a/.size-limit.js b/.size-limit.js index 6e6ee0f68303..00b4bdbfd4d8 100644 --- a/.size-limit.js +++ b/.size-limit.js @@ -240,7 +240,7 @@ module.exports = [ import: createImport('init'), ignore: [...builtinModules, ...nodePrefixedBuiltinModules], gzip: true, - limit: '160 KB', + limit: '161 KB', }, { name: '@sentry/node - without tracing', diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-media-truncation.mjs b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-media-truncation.mjs new file mode 100644 index 000000000000..73891ad30b6f --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-media-truncation.mjs @@ -0,0 +1,79 @@ +import { instrumentAnthropicAiClient } from '@sentry/core'; +import * as Sentry from '@sentry/node'; + +class MockAnthropic { + constructor(config) { + this.apiKey = config.apiKey; + this.baseURL = config.baseURL; + + // Create messages object with create method + this.messages = { + create: this._messagesCreate.bind(this), + }; + } + + /** + * Create a mock message + */ + async _messagesCreate(params) { + // Simulate processing time + await new Promise(resolve => setTimeout(resolve, 10)); + + return { + id: 'msg-truncation-test', + type: 'message', + role: 'assistant', + content: [ + { + type: 'text', + text: 'This is the number **3**.', + }, + ], + model: params.model, + stop_reason: 'end_turn', + stop_sequence: null, + usage: { + input_tokens: 10, + output_tokens: 15, + }, + }; + } +} + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + const mockClient = new MockAnthropic({ + apiKey: 'mock-api-key', + }); + + const client = instrumentAnthropicAiClient(mockClient); + + // Send the image showing the number 3 + await client.messages.create({ + model: 'claude-3-haiku-20240307', + max_tokens: 1024, + messages: [ + { + role: 'user', + content: [ + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: 'base64-mumbo-jumbo'.repeat(100), + }, + }, + ], + }, + { + role: 'user', + content: 'what number is this?', + }, + ], + temperature: 0.7, + }); + }); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts index ebebf60db042..f62975dafb71 100644 --- a/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts @@ -661,4 +661,52 @@ describe('Anthropic integration', () => { }); }, ); + + createEsmAndCjsTests(__dirname, 'scenario-media-truncation.mjs', 'instrument-with-pii.mjs', (createRunner, test) => { + test('truncates media attachment, keeping all other details', async () => { + await createRunner() + .ignore('event') + .expect({ + transaction: { + transaction: 'main', + spans: expect.arrayContaining([ + expect.objectContaining({ + data: expect.objectContaining({ + 'gen_ai.operation.name': 'messages', + 'sentry.op': 'gen_ai.messages', + 'sentry.origin': 'auto.ai.anthropic', + 'gen_ai.system': 'anthropic', + 'gen_ai.request.model': 'claude-3-haiku-20240307', + 'gen_ai.request.messages': JSON.stringify([ + { + role: 'user', + content: [ + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: '[Filtered]', + }, + }, + ], + }, + { + role: 'user', + content: 'what number is this?', + }, + ]), + }), + description: 'messages claude-3-haiku-20240307', + op: 'gen_ai.messages', + origin: 'auto.ai.anthropic', + status: 'ok', + }), + ]), + }, + }) + .start() + .completed(); + }); + }); }); diff --git a/packages/core/src/tracing/ai/messageTruncation.ts b/packages/core/src/tracing/ai/messageTruncation.ts index 945761f6220c..9c8718387404 100644 --- a/packages/core/src/tracing/ai/messageTruncation.ts +++ b/packages/core/src/tracing/ai/messageTruncation.ts @@ -12,13 +12,49 @@ type ContentMessage = { content: string; }; +/** + * Message format used by OpenAI and Anthropic APIs for media. + */ +type ContentArrayMessage = { + [key: string]: unknown; + content: { + [key: string]: unknown; + type: string; + }[]; +}; + +/** + * Inline media content source, with a potentially very large base64 + * blob or data: uri. + */ +type ContentMedia = Record & + ( + | { + media_type: string; + data: string; + } + | { + image_url: `data:${string}`; + } + | { + type: 'blob' | 'base64'; + content: string; + } + | { + b64_json: string; + } + | { + uri: `data:${string}`; + } + ); + /** * Message format used by Google GenAI API. * Parts can be strings or objects with a text property. */ type PartsMessage = { [key: string]: unknown; - parts: Array; + parts: Array; }; /** @@ -26,6 +62,14 @@ type PartsMessage = { */ type TextPart = string | { text: string }; +/** + * A part in a Google GenAI that contains media. + */ +type MediaPart = { + type: string; + content: string; +}; + /** * Calculate the UTF-8 byte length of a string. */ @@ -79,11 +123,12 @@ function truncateTextByBytes(text: string, maxBytes: number): string { * * @returns The text content */ -function getPartText(part: TextPart): string { +function getPartText(part: TextPart | MediaPart): string { if (typeof part === 'string') { return part; } - return part.text; + if ('text' in part) return part.text; + return ''; } /** @@ -93,7 +138,7 @@ function getPartText(part: TextPart): string { * @param text - New text content * @returns New part with updated text */ -function withPartText(part: TextPart, text: string): TextPart { +function withPartText(part: TextPart | MediaPart, text: string): TextPart { if (typeof part === 'string') { return text; } @@ -112,6 +157,43 @@ function isContentMessage(message: unknown): message is ContentMessage { ); } +/** + * Check if a message has the OpenAI/Anthropic content array format. + */ +function isContentArrayMessage(message: unknown): message is ContentArrayMessage { + return message !== null && typeof message === 'object' && 'content' in message && Array.isArray(message.content); +} + +/** + * Check if a content part is an OpenAI/Anthropic media source + */ +function isContentMedia(part: unknown): part is ContentMedia { + if (!part || typeof part !== 'object') return false; + + return ( + isContentMediaSource(part) || + hasInlineData(part) || + ('media_type' in part && typeof part.media_type === 'string' && 'data' in part) || + ('image_url' in part && typeof part.image_url === 'string' && part.image_url.startsWith('data:')) || + ('type' in part && (part.type === 'blob' || part.type === 'base64')) || + 'b64_json' in part || + ('type' in part && 'result' in part && part.type === 'image_generation') || + ('uri' in part && typeof part.uri === 'string' && part.uri.startsWith('data:')) + ); +} +function isContentMediaSource(part: NonNullable): boolean { + return 'type' in part && typeof part.type === 'string' && 'source' in part && isContentMedia(part.source); +} +function hasInlineData(part: NonNullable): part is { inlineData: { data?: string } } { + return ( + 'inlineData' in part && + !!part.inlineData && + typeof part.inlineData === 'object' && + 'data' in part.inlineData && + typeof part.inlineData.data === 'string' + ); +} + /** * Check if a message has the Google GenAI parts format. */ @@ -167,7 +249,7 @@ function truncatePartsMessage(message: PartsMessage, maxBytes: number): unknown[ } // Include parts until we run out of space - const includedParts: TextPart[] = []; + const includedParts: (TextPart | MediaPart)[] = []; for (const part of parts) { const text = getPartText(part); @@ -190,7 +272,14 @@ function truncatePartsMessage(message: PartsMessage, maxBytes: number): unknown[ } } - return includedParts.length > 0 ? [{ ...message, parts: includedParts }] : []; + /* c8 ignore start + * for type safety only, algorithm guarantees SOME text included */ + if (includedParts.length <= 0) { + return []; + } else { + /* c8 ignore stop */ + return [{ ...message, parts: includedParts }]; + } } /** @@ -205,9 +294,11 @@ function truncatePartsMessage(message: PartsMessage, maxBytes: number): unknown[ * @returns Array containing the truncated message, or empty array if truncation fails */ function truncateSingleMessage(message: unknown, maxBytes: number): unknown[] { + /* c8 ignore start - unreachable */ if (!message || typeof message !== 'object') { return []; } + /* c8 ignore stop */ if (isContentMessage(message)) { return truncateContentMessage(message, maxBytes); @@ -221,6 +312,64 @@ function truncateSingleMessage(message: unknown, maxBytes: number): unknown[] { return []; } +const REMOVED_STRING = '[Filtered]'; + +const MEDIA_FIELDS = ['image_url', 'data', 'content', 'b64_json', 'result', 'uri'] as const; + +function stripInlineMediaFromSingleMessage(part: ContentMedia): ContentMedia { + const strip = { ...part }; + if (isContentMedia(strip.source)) { + strip.source = stripInlineMediaFromSingleMessage(strip.source); + } + // google genai inline data blob objects + if (hasInlineData(part)) { + strip.inlineData = { ...part.inlineData, data: REMOVED_STRING }; + } + for (const field of MEDIA_FIELDS) { + if (typeof strip[field] === 'string') strip[field] = REMOVED_STRING; + } + return strip; +} + +/** + * Strip the inline media from message arrays. + * + * This returns a stripped message. We do NOT want to mutate the data in place, + * because of course we still want the actual API/client to handle the media. + */ +function stripInlineMediaFromMessages(messages: unknown[]): unknown[] { + const stripped = messages.map(message => { + let newMessage: Record | undefined = undefined; + if (!!message && typeof message === 'object') { + if (isContentArrayMessage(message)) { + newMessage = { + ...message, + content: stripInlineMediaFromMessages(message.content), + }; + } else if ('content' in message && isContentMedia(message.content)) { + newMessage = { + ...message, + content: stripInlineMediaFromSingleMessage(message.content), + }; + } + if (isPartsMessage(message)) { + newMessage = { + // might have to strip content AND parts + ...(newMessage ?? message), + parts: stripInlineMediaFromMessages(message.parts), + }; + } + if (isContentMedia(newMessage)) { + newMessage = stripInlineMediaFromSingleMessage(newMessage); + } else if (isContentMedia(message)) { + newMessage = stripInlineMediaFromSingleMessage(message); + } + } + return newMessage ?? message; + }); + return stripped; +} + /** * Truncate an array of messages to fit within a byte limit. * @@ -240,26 +389,30 @@ function truncateSingleMessage(message: unknown, maxBytes: number): unknown[] { * // Returns [msg3, msg4] if they fit, or [msg4] if only it fits, etc. * ``` */ -export function truncateMessagesByBytes(messages: unknown[], maxBytes: number): unknown[] { +function truncateMessagesByBytes(messages: unknown[], maxBytes: number): unknown[] { // Early return for empty or invalid input if (!Array.isArray(messages) || messages.length === 0) { return messages; } + // strip inline media first. This will often get us below the threshold, + // while preserving human-readable information about messages sent. + const stripped = stripInlineMediaFromMessages(messages); + // Fast path: if all messages fit, return as-is - const totalBytes = jsonBytes(messages); + const totalBytes = jsonBytes(stripped); if (totalBytes <= maxBytes) { - return messages; + return stripped; } // Precompute each message's JSON size once for efficiency - const messageSizes = messages.map(jsonBytes); + const messageSizes = stripped.map(jsonBytes); // Find the largest suffix (newest messages) that fits within the budget let bytesUsed = 0; - let startIndex = messages.length; // Index where the kept suffix starts + let startIndex = stripped.length; // Index where the kept suffix starts - for (let i = messages.length - 1; i >= 0; i--) { + for (let i = stripped.length - 1; i >= 0; i--) { const messageSize = messageSizes[i]; if (messageSize && bytesUsed + messageSize > maxBytes) { @@ -274,13 +427,14 @@ export function truncateMessagesByBytes(messages: unknown[], maxBytes: number): } // If no complete messages fit, try truncating just the newest message - if (startIndex === messages.length) { - const newestMessage = messages[messages.length - 1]; + if (startIndex === stripped.length) { + // we're truncating down to one message, so all others dropped. + const newestMessage = stripped[stripped.length - 1]; return truncateSingleMessage(newestMessage, maxBytes); } // Return the suffix that fits - return messages.slice(startIndex); + return stripped.slice(startIndex); } /** diff --git a/packages/core/test/lib/tracing/ai-message-truncation.test.ts b/packages/core/test/lib/tracing/ai-message-truncation.test.ts new file mode 100644 index 000000000000..968cd2308bb7 --- /dev/null +++ b/packages/core/test/lib/tracing/ai-message-truncation.test.ts @@ -0,0 +1,336 @@ +import { describe, expect, it } from 'vitest'; +import { truncateGenAiMessages, truncateGenAiStringInput } from '../../../src/tracing/ai/messageTruncation'; + +describe('message truncation utilities', () => { + describe('truncateGenAiMessages', () => { + it('leaves empty/non-array/small messages alone', () => { + // @ts-expect-error - exercising invalid type code path + expect(truncateGenAiMessages(null)).toBe(null); + expect(truncateGenAiMessages([])).toStrictEqual([]); + expect(truncateGenAiMessages([{ text: 'hello' }])).toStrictEqual([{ text: 'hello' }]); + expect(truncateGenAiStringInput('hello')).toBe('hello'); + }); + + it('strips inline media from messages', () => { + const b64 = Buffer.from('lots of data\n').toString('base64'); + const removed = '[Filtered]'; + const messages = [ + { + role: 'user', + content: [ + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: b64, + }, + }, + ], + }, + { + role: 'user', + content: { + image_url: `data:image/png;base64,${b64}`, + }, + }, + { + role: 'agent', + type: 'image', + content: { + b64_json: b64, + }, + }, + { + role: 'system', + inlineData: { + mimeType: 'kiki/booba', + data: 'booboobooboobooba', + }, + content: [ + 'this one has content AND parts and has inline data', + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: b64, + }, + }, + ], + parts: [ + { + inlineData: { + mimeType: 'image/png', + data: 'bloobloobloo', + }, + }, + { + image_url: `data:image/png;base64,${b64}`, + }, + { + type: 'image_generation', + result: b64, + }, + { + uri: `data:image/png;base64,${b64}`, + mediaType: 'image/png', + }, + { + type: 'blob', + mediaType: 'image/png', + content: b64, + }, + { + type: 'text', + text: 'just some text!', + }, + 'unadorned text', + ], + }, + ]; + + // indented json makes for better diffs in test output + const messagesJson = JSON.stringify(messages, null, 2); + const result = truncateGenAiMessages(messages); + + // original messages objects must not be mutated + expect(JSON.stringify(messages, null, 2)).toBe(messagesJson); + expect(result).toStrictEqual([ + { + role: 'user', + content: [ + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: removed, + }, + }, + ], + }, + { + role: 'user', + content: { + image_url: removed, + }, + }, + { + role: 'agent', + type: 'image', + content: { + b64_json: removed, + }, + }, + { + role: 'system', + inlineData: { + mimeType: 'kiki/booba', + data: removed, + }, + content: [ + 'this one has content AND parts and has inline data', + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: removed, + }, + }, + ], + parts: [ + { + inlineData: { + mimeType: 'image/png', + data: removed, + }, + }, + { + image_url: removed, + }, + { + type: 'image_generation', + result: removed, + }, + { + uri: removed, + mediaType: 'image/png', + }, + { + type: 'blob', + mediaType: 'image/png', + content: removed, + }, + { + type: 'text', + text: 'just some text!', + }, + 'unadorned text', + ], + }, + ]); + }); + + const humongous = 'this is a long string '.repeat(10_000); + const giant = 'this is a long string '.repeat(1_000); + const big = 'this is a long string '.repeat(100); + + it('drops older messages to fit in the limit', () => { + const messages = [ + `0 ${giant}`, + { type: 'text', content: `1 ${big}` }, + { type: 'text', content: `2 ${big}` }, + { type: 'text', content: `3 ${giant}` }, + { type: 'text', content: `4 ${big}` }, + `5 ${big}`, + { type: 'text', content: `6 ${big}` }, + { type: 'text', content: `7 ${big}` }, + { type: 'text', content: `8 ${big}` }, + { type: 'text', content: `9 ${big}` }, + { type: 'text', content: `10 ${big}` }, + { type: 'text', content: `11 ${big}` }, + { type: 'text', content: `12 ${big}` }, + ]; + + const messagesJson = JSON.stringify(messages, null, 2); + const result = truncateGenAiMessages(messages); + // should not mutate original messages list + expect(JSON.stringify(messages, null, 2)).toBe(messagesJson); + + // just retain the messages that fit in the budget + expect(result).toStrictEqual([ + `5 ${big}`, + { type: 'text', content: `6 ${big}` }, + { type: 'text', content: `7 ${big}` }, + { type: 'text', content: `8 ${big}` }, + { type: 'text', content: `9 ${big}` }, + { type: 'text', content: `10 ${big}` }, + { type: 'text', content: `11 ${big}` }, + { type: 'text', content: `12 ${big}` }, + ]); + }); + + it('fully drops message if content cannot be made to fit', () => { + const messages = [{ some_other_field: humongous, content: 'hello' }]; + expect(truncateGenAiMessages(messages)).toStrictEqual([]); + }); + + it('truncates if the message content string will not fit', () => { + const messages = [{ content: `2 ${humongous}` }]; + const result = truncateGenAiMessages(messages); + const truncLen = 20_000 - JSON.stringify({ content: '' }).length; + expect(result).toStrictEqual([{ content: `2 ${humongous}`.substring(0, truncLen) }]); + }); + + it('fully drops message if first part overhead does not fit', () => { + const messages = [ + { + parts: [{ some_other_field: humongous }], + }, + ]; + expect(truncateGenAiMessages(messages)).toStrictEqual([]); + }); + + it('fully drops message if overhead too large', () => { + const messages = [ + { + some_other_field: humongous, + parts: [], + }, + ]; + expect(truncateGenAiMessages(messages)).toStrictEqual([]); + }); + + it('truncates if the first message part will not fit', () => { + const messages = [ + { + parts: [`2 ${humongous}`, { some_other_field: 'no text here' }], + }, + ]; + + const result = truncateGenAiMessages(messages); + + // interesting (unexpected?) edge case effect of this truncation. + // subsequent messages count towards truncation overhead limit, + // but are not included, even without their text. This is an edge + // case that seems unlikely in normal usage. + const truncLen = + 20_000 - + JSON.stringify({ + parts: ['', { some_other_field: 'no text here', text: '' }], + }).length; + + expect(result).toStrictEqual([ + { + parts: [`2 ${humongous}`.substring(0, truncLen)], + }, + ]); + }); + + it('truncates if the first message part will not fit, text object', () => { + const messages = [ + { + parts: [{ text: `2 ${humongous}` }], + }, + ]; + const result = truncateGenAiMessages(messages); + const truncLen = + 20_000 - + JSON.stringify({ + parts: [{ text: '' }], + }).length; + expect(result).toStrictEqual([ + { + parts: [ + { + text: `2 ${humongous}`.substring(0, truncLen), + }, + ], + }, + ]); + }); + + it('drops if subsequent message part will not fit, text object', () => { + const messages = [ + { + parts: [ + { text: `1 ${big}` }, + { some_other_field: 'ok' }, + { text: `2 ${big}` }, + { text: `3 ${big}` }, + { text: `4 ${giant}` }, + { text: `5 ${giant}` }, + { text: `6 ${big}` }, + { text: `7 ${big}` }, + { text: `8 ${big}` }, + ], + }, + ]; + const result = truncateGenAiMessages(messages); + expect(result).toStrictEqual([ + { + parts: [{ text: `1 ${big}` }, { some_other_field: 'ok' }, { text: `2 ${big}` }, { text: `3 ${big}` }], + }, + ]); + }); + + it('truncates first message if none fit', () => { + const messages = [{ content: `1 ${humongous}` }, { content: `2 ${humongous}` }, { content: `3 ${humongous}` }]; + const result = truncateGenAiMessages(messages); + const truncLen = 20_000 - JSON.stringify({ content: '' }).length; + expect(result).toStrictEqual([{ content: `3 ${humongous}`.substring(0, truncLen) }]); + }); + + it('drops if first message cannot be safely truncated', () => { + const messages = [ + { content: `1 ${humongous}` }, + { content: `2 ${humongous}` }, + { what_even_is_this: `? ${humongous}` }, + ]; + const result = truncateGenAiMessages(messages); + expect(result).toStrictEqual([]); + }); + }); +});