feat(core): Improve error handling for Anthropic AI instrumentation (#17535)

This PR enhances Anthropic AI instrumentation to properly handle and
record errors that occur as part of response metadata.

Core Error Handling Improvements:

- Enhanced error detection: Added proper handling for Anthropic API
error responses with type: 'error' structure
- Improved streaming error handling: Better error type detection and
reporting for streaming operations
- Error capturing: Manually captured errors for both standard and
streaming errors

Author: RulaKhaled

dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-errors.mjs

@@ -0,0 +1,115 @@
+import { instrumentAnthropicAiClient } from '@sentry/core';
+import * as Sentry from '@sentry/node';
+
+class MockAnthropic {
+  constructor(config) {
+    this.apiKey = config.apiKey;
+    this.messages = {
+      create: this._messagesCreate.bind(this),
+    };
+    this.models = {
+      retrieve: this._modelsRetrieve.bind(this),
+    };
+  }
+
+  async _messagesCreate(params) {
+    await new Promise(resolve => setTimeout(resolve, 5));
+
+    // Case 1: Invalid tool format error
+    if (params.model === 'invalid-format') {
+      const error = new Error('Invalid format');
+      error.status = 400;
+      error.headers = { 'x-request-id': 'mock-invalid-tool-format-error' };
+      throw error;
+    }
+
+    // Default case (success) - return tool use for successful tool usage test
+    return {
+      id: 'msg_ok',
+      type: 'message',
+      model: params.model,
+      role: 'assistant',
+      content: [
+        {
+          type: 'tool_use',
+          id: 'tool_ok_1',
+          name: 'calculator',
+          input: { expression: '2+2' },
+        },
+      ],
+      stop_reason: 'tool_use',
+      usage: { input_tokens: 7, output_tokens: 9 },
+    };
+  }
+
+  async _modelsRetrieve(modelId) {
+    await new Promise(resolve => setTimeout(resolve, 5));
+
+    // Case for model retrieval error
+    if (modelId === 'nonexistent-model') {
+      const error = new Error('Model not found');
+      error.status = 404;
+      error.headers = { 'x-request-id': 'mock-model-retrieval-error' };
+      throw error;
+    }
+
+    return {
+      id: modelId,
+      name: modelId,
+      created_at: 1715145600,
+      model: modelId,
+    };
+  }
+}
+
+async function run() {
+  await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
+    const mockClient = new MockAnthropic({ apiKey: 'mock-api-key' });
+    const client = instrumentAnthropicAiClient(mockClient);
+
+    // 1. Test invalid format error
+    // https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/implement-tool-use#handling-tool-use-and-tool-result-content-blocks
+    try {
+      await client.messages.create({
+        model: 'invalid-format',
+        messages: [
+          {
+            role: 'user',
+            content: [
+              { type: 'text', text: 'Here are the results:' }, // ❌ Text before tool_result
+              { type: 'tool_result', tool_use_id: 'toolu_01' },
+            ],
+          },
+        ],
+      });
+    } catch {
+      // Error expected
+    }
+
+    // 2. Test model retrieval error
+    try {
+      await client.models.retrieve('nonexistent-model');
+    } catch {
+      // Error expected
+    }
+
+    // 3. Test successful tool usage for comparison
+    await client.messages.create({
+      model: 'claude-3-haiku-20240307',
+      messages: [{ role: 'user', content: 'Calculate 2+2' }],
+      tools: [
+        {
+          name: 'calculator',
+          description: 'Perform calculations',
+          input_schema: {
+            type: 'object',
+            properties: { expression: { type: 'string' } },
+            required: ['expression'],
+          },
+        },
+      ],
+    });
+  });
+}
+
+run();

dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-stream-errors.mjs

@@ -0,0 +1,166 @@
+import { instrumentAnthropicAiClient } from '@sentry/core';
+import * as Sentry from '@sentry/node';
+
+// Generator for default fallback
+function createMockDefaultFallbackStream() {
+  async function* generator() {
+    yield {
+      type: 'content_block_start',
+      index: 0,
+    };
+    yield {
+      type: 'content_block_delta',
+      index: 0,
+      delta: { text: 'This stream will work fine.' },
+    };
+    yield {
+      type: 'content_block_stop',
+      index: 0,
+    };
+  }
+  return generator();
+}
+
+// Generator that errors midway through streaming
+function createMockMidwayErrorStream() {
+  async function* generator() {
+    // First yield some initial data to start the stream
+    yield {
+      type: 'content_block_start',
+      message: {
+        id: 'msg_error_stream_1',
+        type: 'message',
+        role: 'assistant',
+        model: 'claude-3-haiku-20240307',
+        content: [],
+        usage: { input_tokens: 5 },
+      },
+    };
+
+    // Yield one chunk of content
+    yield { type: 'content_block_delta', delta: { text: 'This stream will ' } };
+
+    // Then throw an error
+    await new Promise(resolve => setTimeout(resolve, 5));
+    throw new Error('Stream interrupted');
+  }
+
+  return generator();
+}
+
+class MockAnthropic {
+  constructor(config) {
+    this.apiKey = config.apiKey;
+
+    this.messages = {
+      create: this._messagesCreate.bind(this),
+      stream: this._messagesStream.bind(this),
+    };
+  }
+
+  // client.messages.create with stream: true
+  async _messagesCreate(params) {
+    await new Promise(resolve => setTimeout(resolve, 5));
+
+    // Error on initialization for 'error-stream-init' model
+    if (params.model === 'error-stream-init') {
+      if (params?.stream === true) {
+        throw new Error('Failed to initialize stream');
+      }
+    }
+
+    // Error midway for 'error-stream-midway' model
+    if (params.model === 'error-stream-midway') {
+      if (params?.stream === true) {
+        return createMockMidwayErrorStream();
+      }
+    }
+
+    // Default fallback
+    return {
+      id: 'msg_mock123',
+      type: 'message',
+      model: params.model,
+      role: 'assistant',
+      content: [{ type: 'text', text: 'Non-stream response' }],
+      usage: { input_tokens: 5, output_tokens: 7 },
+    };
+  }
+
+  // client.messages.stream
+  async _messagesStream(params) {
+    await new Promise(resolve => setTimeout(resolve, 5));
+
+    // Error on initialization for 'error-stream-init' model
+    if (params.model === 'error-stream-init') {
+      throw new Error('Failed to initialize stream');
+    }
+
+    // Error midway for 'error-stream-midway' model
+    if (params.model === 'error-stream-midway') {
+      return createMockMidwayErrorStream();
+    }
+
+    // Default fallback
+    return createMockDefaultFallbackStream();
+  }
+}
+
+async function run() {
+  await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
+    const mockClient = new MockAnthropic({ apiKey: 'mock-api-key' });
+    const client = instrumentAnthropicAiClient(mockClient);
+
+    // 1) Error on stream initialization with messages.create
+    try {
+      await client.messages.create({
+        model: 'error-stream-init',
+        messages: [{ role: 'user', content: 'This will fail immediately' }],
+        stream: true,
+      });
+    } catch {
+      // Error expected
+    }
+
+    // 2) Error on stream initialization with messages.stream
+    try {
+      await client.messages.stream({
+        model: 'error-stream-init',
+        messages: [{ role: 'user', content: 'This will also fail immediately' }],
+      });
+    } catch {
+      // Error expected
+    }
+
+    // 3) Error midway through streaming with messages.create
+    try {
+      const stream = await client.messages.create({
+        model: 'error-stream-midway',
+        messages: [{ role: 'user', content: 'This will fail midway' }],
+        stream: true,
+      });
+
+      for await (const _ of stream) {
+        void _;
+      }
+    } catch {
+      // Error expected
+    }
+
+    // 4) Error midway through streaming with messages.stream
+    try {
+      const stream = await client.messages.stream({
+        model: 'error-stream-midway',
+        messages: [{ role: 'user', content: 'This will also fail midway' }],
+      });
+
+      for await (const _ of stream) {
+        void _;
+      }
+    } catch {
+      // Error expected
+    }
+  });
+}
+
+run();

dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts

@@ -348,4 +348,101 @@ describe('Anthropic integration', () => {
         .completed();
     });
   });
+
+  // Additional error scenarios - Streaming errors
+  const EXPECTED_STREAM_ERROR_SPANS = {
+    transaction: 'main',
+    spans: expect.arrayContaining([
+      // Error with messages.create on stream initialization
+      expect.objectContaining({
+        description: 'messages error-stream-init stream-response',
+        op: 'gen_ai.messages',
+        status: 'internal_error', // Actual status coming from the instrumentation
+        data: expect.objectContaining({
+          'gen_ai.request.model': 'error-stream-init',
+          'gen_ai.request.stream': true,
+        }),
+      }),
+      // Error with messages.stream on stream initialization
+      expect.objectContaining({
+        description: 'messages error-stream-init stream-response',
+        op: 'gen_ai.messages',
+        status: 'internal_error', // Actual status coming from the instrumentation
+        data: expect.objectContaining({
+          'gen_ai.request.model': 'error-stream-init',
+        }),
+      }),
+      // Error midway with messages.create on streaming - note: The stream is started successfully
+      // so we get a successful span with the content that was streamed before the error
+      expect.objectContaining({
+        description: 'messages error-stream-midway stream-response',
+        op: 'gen_ai.messages',
+        status: 'ok',
+        data: expect.objectContaining({
+          'gen_ai.request.model': 'error-stream-midway',
+          'gen_ai.request.stream': true,
+          'gen_ai.response.streaming': true,
+          'gen_ai.response.text': 'This stream will ', // We received some data before error
+        }),
+      }),
+      // Error midway with messages.stream - same behavior, we get a span with the streamed data
+      expect.objectContaining({
+        description: 'messages error-stream-midway stream-response',
+        op: 'gen_ai.messages',
+        status: 'ok',
+        data: expect.objectContaining({
+          'gen_ai.request.model': 'error-stream-midway',
+          'gen_ai.response.streaming': true,
+          'gen_ai.response.text': 'This stream will ', // We received some data before error
+        }),
+      }),
+    ]),
+  };
+
+  createEsmAndCjsTests(__dirname, 'scenario-stream-errors.mjs', 'instrument-with-pii.mjs', (createRunner, test) => {
+    test('handles streaming errors correctly', async () => {
+      await createRunner().ignore('event').expect({ transaction: EXPECTED_STREAM_ERROR_SPANS }).start().completed();
+    });
+  });
+
+  // Additional error scenarios - Tool errors and model retrieval errors
+  const EXPECTED_ERROR_SPANS = {
+    transaction: 'main',
+    spans: expect.arrayContaining([
+      // Invalid tool format error
+      expect.objectContaining({
+        description: 'messages invalid-format',
+        op: 'gen_ai.messages',
+        status: 'unknown_error',
+        data: expect.objectContaining({
+          'gen_ai.request.model': 'invalid-format',
+        }),
+      }),
+      // Model retrieval error
+      expect.objectContaining({
+        description: 'models nonexistent-model',
+        op: 'gen_ai.models',
+        status: 'unknown_error',
+        data: expect.objectContaining({
+          'gen_ai.request.model': 'nonexistent-model',
+        }),
+      }),
+      // Successful tool usage (for comparison)
+      expect.objectContaining({
+        description: 'messages claude-3-haiku-20240307',
+        op: 'gen_ai.messages',
+        status: 'ok',
+        data: expect.objectContaining({
+          'gen_ai.request.model': 'claude-3-haiku-20240307',
+          'gen_ai.response.tool_calls': expect.stringContaining('tool_ok_1'),
+        }),
+      }),
+    ]),
+  };
+
+  createEsmAndCjsTests(__dirname, 'scenario-errors.mjs', 'instrument-with-pii.mjs', (createRunner, test) => {
+    test('handles tool errors and model retrieval errors correctly', async () => {
+      await createRunner().ignore('event').expect({ transaction: EXPECTED_ERROR_SPANS }).start().completed();
+    });
+  });
 });