From 0c1e771a98d1714343b771900c302e2d2af8325b Mon Sep 17 00:00:00 2001 From: adaines Date: Wed, 5 Nov 2025 13:14:44 -0500 Subject: [PATCH 1/7] feat: add support for ChatOptions.ResponseFormat in AWSSDK.Extensions.Bedrock.MEAI --- .../BedrockChatClient.cs | 242 ++++- .../BedrockChatClientTests.cs | 911 +++++++++++++++++- 2 files changed, 1135 insertions(+), 18 deletions(-) diff --git a/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/BedrockChatClient.cs b/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/BedrockChatClient.cs index bfe33dbda368..1c95ce8a5402 100644 --- a/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/BedrockChatClient.cs +++ b/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/BedrockChatClient.cs @@ -18,8 +18,10 @@ using Amazon.Runtime.Internal.Util; using Microsoft.Extensions.AI; using System; +using System.Buffers; using System.Collections.Generic; using System.Diagnostics; +using System.IO; using System.Linq; using System.Runtime.CompilerServices; using System.Text; @@ -35,6 +37,13 @@ internal sealed partial class BedrockChatClient : IChatClient /// A default logger to use. private static readonly ILogger DefaultLogger = Logger.GetLogger(typeof(BedrockChatClient)); + /// The name used for the synthetic tool that enforces response format. + private const string ResponseFormatToolName = "generate_response"; + /// The description used for the synthetic tool that enforces response format. + private const string ResponseFormatToolDescription = "Generate response in specified format"; + /// Maximum nesting depth for Document to JSON conversion to prevent stack overflow. + private const int MaxDocumentNestingDepth = 100; + /// The wrapped instance. private readonly IAmazonBedrockRuntime _runtime; /// Default model ID to use when no model is specified in the request. @@ -42,11 +51,7 @@ internal sealed partial class BedrockChatClient : IChatClient /// Metadata describing the chat client. private readonly ChatClientMetadata _metadata; - /// - /// Initializes a new instance of the class. - /// - /// The instance to wrap. - /// Model ID to use as the default when no model ID is specified in a request. + /// Initializes a new instance of the class. public BedrockChatClient(IAmazonBedrockRuntime runtime, string? defaultModelId) { Debug.Assert(runtime is not null); @@ -79,7 +84,34 @@ public async Task GetResponseAsync( request.InferenceConfig = CreateInferenceConfiguration(request.InferenceConfig, options); request.AdditionalModelRequestFields = CreateAdditionalModelRequestFields(request.AdditionalModelRequestFields, options); - var response = await _runtime.ConverseAsync(request, cancellationToken).ConfigureAwait(false); + // Execute the request with proper error handling for ResponseFormat scenarios + ConverseResponse response; + try + { + response = await _runtime.ConverseAsync(request, cancellationToken).ConfigureAwait(false); + } + catch (AmazonBedrockRuntimeException ex) when (options?.ResponseFormat is ChatResponseFormatJson) + { + // Check if this is a ToolChoice validation error (model doesn't support it) + bool isToolChoiceNotSupported = + ex.ErrorCode == "ValidationException" && + (ex.Message.IndexOf("toolChoice", StringComparison.OrdinalIgnoreCase) >= 0 || + ex.Message.IndexOf("tool_choice", StringComparison.OrdinalIgnoreCase) >= 0 || + ex.Message.IndexOf("ToolChoice", StringComparison.OrdinalIgnoreCase) >= 0); + + if (isToolChoiceNotSupported) + { + // Provide a more helpful error message when ToolChoice fails due to model limitations + throw new NotSupportedException( + $"The model '{request.ModelId}' does not support ResponseFormat. " + + $"ResponseFormat requires ToolChoice support, which is only available in Claude 3+ and Mistral Large models. " + + $"See: https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference-supported-models-features.html", + ex); + } + + // Re-throw other exceptions as-is + throw; + } ChatMessage result = new() { @@ -89,6 +121,50 @@ public async Task GetResponseAsync( MessageId = Guid.NewGuid().ToString("N"), }; + // Check if ResponseFormat was used and extract structured content + bool usingResponseFormat = options?.ResponseFormat is ChatResponseFormatJson; + if (usingResponseFormat) + { + var structuredContent = ExtractResponseFormatContent(response.Output?.Message); + if (structuredContent is not null) + { + // Replace the content with the extracted JSON as a TextContent + result.Contents.Add(new TextContent(structuredContent) { RawRepresentation = response.Output?.Message }); + + // Skip normal content processing since we've extracted the structured response + if (DocumentToDictionary(response.AdditionalModelResponseFields) is { } responseFieldsDict) + { + result.AdditionalProperties = new(responseFieldsDict); + } + + return new(result) + { + CreatedAt = result.CreatedAt, + FinishReason = response.StopReason is not null ? GetChatFinishReason(response.StopReason) : null, + Usage = response.Usage is TokenUsage tokenUsage ? CreateUsageDetails(tokenUsage) : null, + RawRepresentation = response, + }; + } + else + { + // User requested structured output but didn't get it - this is a contract violation + var errorMessage = string.Format( + "ResponseFormat was specified but model did not return expected tool use. ModelId: {0}, StopReason: {1}", + request.ModelId, + response.StopReason?.Value ?? "unknown"); + + DefaultLogger.Error(new InvalidOperationException(errorMessage), errorMessage); + + // Always throw when ResponseFormat was requested but not fulfilled + throw new InvalidOperationException( + $"Model '{request.ModelId}' did not return structured output as requested. " + + $"This may indicate the model refused to follow the tool use instruction, " + + $"the schema was too complex, or the prompt conflicted with the requirement. " + + $"StopReason: {response.StopReason?.Value ?? "unknown"}"); + } + } + + // Normal content processing when not using ResponseFormat or extraction failed if (response.Output?.Message?.Content is { } contents) { foreach (var content in contents) @@ -182,6 +258,14 @@ public async IAsyncEnumerable GetStreamingResponseAsync( throw new ArgumentNullException(nameof(messages)); } + // Check if ResponseFormat is set - not supported for streaming yet + if (options?.ResponseFormat is ChatResponseFormatJson) + { + throw new NotSupportedException( + "ResponseFormat is not yet supported for streaming responses with Amazon Bedrock. " + + "Please use GetResponseAsync for structured output."); + } + ConverseStreamRequest request = options?.RawRepresentationFactory?.Invoke(this) as ConverseStreamRequest ?? new(); request.ModelId ??= options?.ModelId ?? _modelId; request.Messages = CreateMessages(request.Messages, messages); @@ -794,7 +878,11 @@ private static Document ToDocument(JsonElement json) } } - /// Creates an from the specified options. + /// Creates a from the specified options. + /// + /// When ResponseFormat is specified, creates a synthetic tool to enforce structured output. + /// This conflicts with user-provided tools as Bedrock only supports a single ToolChoice value. + /// private static ToolConfiguration? CreateToolConfig(ToolConfiguration? toolConfig, ChatOptions? options) { if (options?.Tools is { Count: > 0 } tools) @@ -857,6 +945,56 @@ private static Document ToDocument(JsonElement json) } } + // Handle ResponseFormat by creating a synthetic tool + if (options?.ResponseFormat is ChatResponseFormatJson jsonFormat) + { + // Check for conflict with user-provided tools + if (toolConfig?.Tools?.Count > 0) + { + throw new ArgumentException( + "ResponseFormat cannot be used with Tools in Amazon Bedrock. " + + "ResponseFormat uses Bedrock's tool mechanism for structured output, " + + "which conflicts with user-provided tools."); + } + + // Create the synthetic tool with the schema from ResponseFormat + toolConfig ??= new(); + toolConfig.Tools ??= []; + + // Parse the schema if provided, otherwise create an empty object schema + Document schemaDoc; + if (jsonFormat.Schema.HasValue) + { + // Schema is already a JsonElement (parsed JSON), convert directly to Document + schemaDoc = ToDocument(jsonFormat.Schema.Value); + } + else + { + // For JSON mode without schema, create a generic object schema + schemaDoc = new Document(new Dictionary + { + ["type"] = new Document("object"), + ["additionalProperties"] = new Document(true) + }); + } + + toolConfig.Tools.Add(new Tool + { + ToolSpec = new ToolSpecification + { + Name = ResponseFormatToolName, + Description = jsonFormat.SchemaDescription ?? ResponseFormatToolDescription, + InputSchema = new ToolInputSchema + { + Json = schemaDoc + } + } + }); + + // Force the model to use the synthetic tool + toolConfig.ToolChoice = new ToolChoice { Tool = new() { Name = ResponseFormatToolName } }; + } + if (toolConfig?.Tools is { Count: > 0 } && toolConfig.ToolChoice is null) { switch (options!.ToolMode) @@ -872,6 +1010,96 @@ private static Document ToDocument(JsonElement json) return toolConfig; } + /// Extracts JSON content from the synthetic ResponseFormat tool use, if present. + private static string? ExtractResponseFormatContent(Message? message) + { + if (message?.Content is null) + { + return null; + } + + foreach (var content in message.Content) + { + if (content.ToolUse is ToolUseBlock toolUse && + toolUse.Name == ResponseFormatToolName && + toolUse.Input != default) + { + // Convert the Document back to JSON string + return DocumentToJsonString(toolUse.Input); + } + } + + return null; + } + + /// Converts a to a JSON string. + private static string DocumentToJsonString(Document document) + { + using var stream = new MemoryStream(); + using (var writer = new Utf8JsonWriter(stream, new JsonWriterOptions { Indented = false })) + { + WriteDocumentAsJson(writer, document); + } // Explicit scope to ensure writer is flushed before reading buffer + + return Encoding.UTF8.GetString(stream.ToArray()); + } + + /// Recursively writes a as JSON. + private static void WriteDocumentAsJson(Utf8JsonWriter writer, Document document, int depth = 0) + { + // Check depth to prevent stack overflow from deeply nested or circular structures + if (depth > MaxDocumentNestingDepth) + { + throw new InvalidOperationException( + $"Document nesting depth exceeds maximum of {MaxDocumentNestingDepth}. " + + $"This may indicate a circular reference or excessively nested data structure."); + } + + if (document.IsBool()) + { + writer.WriteBooleanValue(document.AsBool()); + } + else if (document.IsInt()) + { + writer.WriteNumberValue(document.AsInt()); + } + else if (document.IsLong()) + { + writer.WriteNumberValue(document.AsLong()); + } + else if (document.IsDouble()) + { + writer.WriteNumberValue(document.AsDouble()); + } + else if (document.IsString()) + { + writer.WriteStringValue(document.AsString()); + } + else if (document.IsDictionary()) + { + writer.WriteStartObject(); + foreach (var kvp in document.AsDictionary()) + { + writer.WritePropertyName(kvp.Key); + WriteDocumentAsJson(writer, kvp.Value, depth + 1); + } + writer.WriteEndObject(); + } + else if (document.IsList()) + { + writer.WriteStartArray(); + foreach (var item in document.AsList()) + { + WriteDocumentAsJson(writer, item, depth + 1); + } + writer.WriteEndArray(); + } + else + { + writer.WriteNullValue(); + } + } + /// Creates an from the specified options. private static InferenceConfiguration CreateInferenceConfiguration(InferenceConfiguration config, ChatOptions? options) { diff --git a/extensions/test/BedrockMEAITests/BedrockChatClientTests.cs b/extensions/test/BedrockMEAITests/BedrockChatClientTests.cs index 8f5099c973d8..58473a8438a3 100644 --- a/extensions/test/BedrockMEAITests/BedrockChatClientTests.cs +++ b/extensions/test/BedrockMEAITests/BedrockChatClientTests.cs @@ -1,11 +1,111 @@ -using Microsoft.Extensions.AI; +using Amazon.BedrockRuntime.Model; +using Amazon.Runtime.Documents; +using Microsoft.Extensions.AI; using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; using Xunit; namespace Amazon.BedrockRuntime; +// Mock implementation to capture requests and control responses +internal sealed class MockBedrockRuntime : IAmazonBedrockRuntime +{ + public ConverseRequest CapturedRequest { get; private set; } + public ConverseStreamRequest CapturedStreamRequest { get; private set; } + public Func ResponseFactory { get; set; } + public Exception ExceptionToThrow { get; set; } + + public Task ConverseAsync(ConverseRequest request, CancellationToken cancellationToken = default) + { + CapturedRequest = request; + + if (ExceptionToThrow != null) + { + throw ExceptionToThrow; + } + + if (ResponseFactory != null) + { + return Task.FromResult(ResponseFactory(request)); + } + + // Default response + return Task.FromResult(new ConverseResponse + { + Output = new ConverseOutput + { + Message = new Message + { + Role = ConversationRole.Assistant, + Content = new List + { + new ContentBlock { Text = "Default response" } + } + } + }, + StopReason = new StopReason("end_turn") + }); + } + + public Task ConverseStreamAsync(ConverseStreamRequest request, CancellationToken cancellationToken = default) + { + CapturedStreamRequest = request; + throw new NotImplementedException("Stream testing not implemented in this mock"); + } + + public void Dispose() { } + + // Unused interface members - all throw NotImplementedException + public IBedrockRuntimePaginatorFactory Paginators => throw new NotImplementedException(); + public Amazon.Runtime.IClientConfig Config => throw new NotImplementedException(); + + // Sync methods + public ApplyGuardrailResponse ApplyGuardrail(ApplyGuardrailRequest request) => throw new NotImplementedException(); + public ConverseResponse Converse(ConverseRequest request) => throw new NotImplementedException(); + public ConverseStreamResponse ConverseStream(ConverseStreamRequest request) => throw new NotImplementedException(); + public CountTokensResponse CountTokens(CountTokensRequest request) => throw new NotImplementedException(); + public GetAsyncInvokeResponse GetAsyncInvoke(GetAsyncInvokeRequest request) => throw new NotImplementedException(); + public InvokeModelResponse InvokeModel(InvokeModelRequest request) => throw new NotImplementedException(); + public InvokeModelWithResponseStreamResponse InvokeModelWithResponseStream(InvokeModelWithResponseStreamRequest request) => throw new NotImplementedException(); + public ListAsyncInvokesResponse ListAsyncInvokes(ListAsyncInvokesRequest request) => throw new NotImplementedException(); + public StartAsyncInvokeResponse StartAsyncInvoke(StartAsyncInvokeRequest request) => throw new NotImplementedException(); + + // Async methods + public Task ApplyGuardrailAsync(ApplyGuardrailRequest request, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task CountTokensAsync(CountTokensRequest request, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task GetAsyncInvokeAsync(GetAsyncInvokeRequest request, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task InvokeModelAsync(InvokeModelRequest request, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task InvokeModelWithResponseStreamAsync(InvokeModelWithResponseStreamRequest request, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task ListAsyncInvokesAsync(ListAsyncInvokesRequest request, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + public Task StartAsyncInvokeAsync(StartAsyncInvokeRequest request, CancellationToken cancellationToken = default) => throw new NotImplementedException(); + + // Endpoint determination + public Amazon.Runtime.Endpoints.Endpoint DetermineServiceOperationEndpoint(Amazon.Runtime.AmazonWebServiceRequest request) => throw new NotImplementedException(); +} + +// Simple test implementation of AIFunctionDeclaration +internal sealed class TestAIFunction : AIFunctionDeclaration +{ + public TestAIFunction(string name, string description, JsonElement jsonSchema) + { + Name = name; + Description = description; + JsonSchema = jsonSchema; + } + + public override string Name { get; } + public override string Description { get; } + public override JsonElement JsonSchema { get; } +} + public class BedrockChatClientTests { + #region Basic Client Tests + [Fact] [Trait("UnitTest", "BedrockRuntime")] public void AsIChatClient_InvalidArguments_Throws() @@ -19,8 +119,8 @@ public void AsIChatClient_InvalidArguments_Throws() [InlineData("claude")] public void AsIChatClient_ReturnsInstance(string modelId) { - IAmazonBedrockRuntime runtime = new AmazonBedrockRuntimeClient("awsAccessKeyId", "awsSecretAccessKey", RegionEndpoint.USEast1); - IChatClient client = runtime.AsIChatClient(modelId); + var mock = new MockBedrockRuntime(); + IChatClient client = mock.AsIChatClient(modelId); Assert.NotNull(client); Assert.Equal("aws.bedrock", client.GetService()?.ProviderName); @@ -31,17 +131,806 @@ public void AsIChatClient_ReturnsInstance(string modelId) [Trait("UnitTest", "BedrockRuntime")] public void AsIChatClient_GetService() { - IAmazonBedrockRuntime runtime = new AmazonBedrockRuntimeClient("awsAccessKeyId", "awsSecretAccessKey", RegionEndpoint.USEast1); - IChatClient client = runtime.AsIChatClient(); + var mock = new MockBedrockRuntime(); + IChatClient client = mock.AsIChatClient(); - Assert.Same(runtime, client.GetService()); - Assert.Same(runtime, client.GetService()); + Assert.Same(mock, client.GetService()); Assert.Same(client, client.GetService()); - Assert.Null(client.GetService()); - - Assert.Null(client.GetService("key")); Assert.Null(client.GetService("key")); - Assert.Null(client.GetService("key")); } + + #endregion + + #region Synthetic Tool Creation Tests + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Json_WithoutSchema_CreatesSyntheticToolWithGenericObjectSchema() + { + // Arrange + var mock = new MockBedrockRuntime(); + var client = mock.AsIChatClient("claude-3"); + var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; + var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; + + // Act + try + { + await client.GetResponseAsync(messages, options); + } + catch + { + // We're testing request creation, not response processing + } + + // Assert - Verify synthetic tool was created correctly + Assert.NotNull(mock.CapturedRequest); + Assert.NotNull(mock.CapturedRequest.ToolConfig); + Assert.NotNull(mock.CapturedRequest.ToolConfig.Tools); + Assert.Single(mock.CapturedRequest.ToolConfig.Tools); + + var tool = mock.CapturedRequest.ToolConfig.Tools[0]; + Assert.NotNull(tool.ToolSpec); + Assert.Equal("generate_response", tool.ToolSpec.Name); + Assert.Equal("Generate response in specified format", tool.ToolSpec.Description); + + // Verify schema is generic object schema + var schema = tool.ToolSpec.InputSchema.Json; + Assert.True(schema.IsDictionary()); + var schemaDict = schema.AsDictionary(); + Assert.True(schemaDict.ContainsKey("type")); + Assert.Equal("object", schemaDict["type"].AsString()); + Assert.True(schemaDict.ContainsKey("additionalProperties")); + Assert.True(schemaDict["additionalProperties"].AsBool()); + + // Verify ToolChoice forces the synthetic tool + Assert.NotNull(mock.CapturedRequest.ToolConfig.ToolChoice); + Assert.NotNull(mock.CapturedRequest.ToolConfig.ToolChoice.Tool); + Assert.Equal("generate_response", mock.CapturedRequest.ToolConfig.ToolChoice.Tool.Name); + } + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Json_WithSchema_CreatesSyntheticToolWithCorrectSchema() + { + // Arrange + var mock = new MockBedrockRuntime(); + var client = mock.AsIChatClient("claude-3"); + var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; + + var schemaJson = """ + { + "type": "object", + "properties": { + "name": { "type": "string" }, + "age": { "type": "number" } + }, + "required": ["name"] + } + """; + var schemaElement = JsonDocument.Parse(schemaJson).RootElement; + var options = new ChatOptions + { + ResponseFormat = ChatResponseFormat.ForJsonSchema(schemaElement, + schemaName: "PersonSchema", + schemaDescription: "A person object") + }; + + // Act + try + { + await client.GetResponseAsync(messages, options); + } + catch + { + // We're testing request creation + } + + // Assert + var tool = mock.CapturedRequest.ToolConfig.Tools[0]; + Assert.Equal("generate_response", tool.ToolSpec.Name); + Assert.Equal("A person object", tool.ToolSpec.Description); + + // Verify schema structure matches input + var schema = tool.ToolSpec.InputSchema.Json; + Assert.True(schema.IsDictionary()); + var schemaDict = schema.AsDictionary(); + + Assert.Equal("object", schemaDict["type"].AsString()); + Assert.True(schemaDict.ContainsKey("properties")); + + var properties = schemaDict["properties"].AsDictionary(); + Assert.True(properties.ContainsKey("name")); + Assert.True(properties.ContainsKey("age")); + Assert.Equal("string", properties["name"].AsDictionary()["type"].AsString()); + Assert.Equal("number", properties["age"].AsDictionary()["type"].AsString()); + + Assert.True(schemaDict.ContainsKey("required")); + var required = schemaDict["required"].AsList(); + Assert.Single(required); + Assert.Equal("name", required[0].AsString()); + } + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Json_WithTools_ThrowsArgumentException() + { + // Arrange + var mock = new MockBedrockRuntime(); + var client = mock.AsIChatClient("claude-3"); + var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; + + var toolSchema = JsonDocument.Parse("""{"type": "object"}""").RootElement; + var options = new ChatOptions + { + ResponseFormat = ChatResponseFormat.Json, + Tools = new List + { + new TestAIFunction("test_tool", "Test", toolSchema) + } + }; + + // Act & Assert + var ex = await Assert.ThrowsAsync(async () => + await client.GetResponseAsync(messages, options)); + + Assert.Contains("ResponseFormat cannot be used with Tools", ex.Message); + Assert.Contains("Bedrock's tool mechanism", ex.Message); + } + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Text_DoesNotCreateSyntheticTool() + { + // Arrange + var mock = new MockBedrockRuntime(); + var client = mock.AsIChatClient("claude-3"); + var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; + var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Text }; + + // Act + await client.GetResponseAsync(messages, options); + + // Assert - No synthetic tool should be created for Text format + Assert.True(mock.CapturedRequest.ToolConfig == null || + mock.CapturedRequest.ToolConfig.Tools == null || + mock.CapturedRequest.ToolConfig.Tools.Count == 0); + } + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_NotSet_DoesNotCreateSyntheticTool() + { + // Arrange + var mock = new MockBedrockRuntime(); + var client = mock.AsIChatClient("claude-3"); + var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; + + // Act + await client.GetResponseAsync(messages); + + // Assert + Assert.True(mock.CapturedRequest.ToolConfig == null || + mock.CapturedRequest.ToolConfig.Tools == null || + mock.CapturedRequest.ToolConfig.Tools.Count == 0); + } + + #endregion + + #region Response Extraction Tests + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Json_ModelReturnsToolUse_ExtractsJsonCorrectly() + { + // Arrange + var mock = new MockBedrockRuntime(); + var client = mock.AsIChatClient("claude-3"); + var messages = new[] { new ChatMessage(ChatRole.User, "Get weather") }; + var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; + + // Setup mock to return tool use with structured data + mock.ResponseFactory = req => new ConverseResponse + { + Output = new ConverseOutput + { + Message = new Message + { + Role = ConversationRole.Assistant, + Content = new List + { + new ContentBlock + { + ToolUse = new ToolUseBlock + { + ToolUseId = "test-id", + Name = "generate_response", + Input = new Document(new Dictionary + { + ["city"] = new Document("Seattle"), + ["temperature"] = new Document(72), + ["conditions"] = new Document("sunny") + }) + } + } + } + } + }, + StopReason = new StopReason("tool_use"), + Usage = new TokenUsage { InputTokens = 10, OutputTokens = 20, TotalTokens = 30 } + }; + + // Act + var response = await client.GetResponseAsync(messages, options); + + // Assert + Assert.NotNull(response); + Assert.NotNull(response.Text); + + // Parse the JSON to verify structure + var json = JsonDocument.Parse(response.Text); + Assert.Equal("Seattle", json.RootElement.GetProperty("city").GetString()); + Assert.Equal(72, json.RootElement.GetProperty("temperature").GetInt32()); + Assert.Equal("sunny", json.RootElement.GetProperty("conditions").GetString()); + } + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Json_ModelReturnsComplexNestedData_ExtractsCorrectly() + { + // Arrange + var mock = new MockBedrockRuntime(); + var client = mock.AsIChatClient("claude-3"); + var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; + var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; + + // Setup complex nested structure + mock.ResponseFactory = req => new ConverseResponse + { + Output = new ConverseOutput + { + Message = new Message + { + Role = ConversationRole.Assistant, + Content = new List + { + new ContentBlock + { + ToolUse = new ToolUseBlock + { + ToolUseId = "test-id", + Name = "generate_response", + Input = new Document(new Dictionary + { + ["user"] = new Document(new Dictionary + { + ["name"] = new Document("John"), + ["age"] = new Document(30), + ["active"] = new Document(true) + }), + ["scores"] = new Document(new Document[] + { + new Document(85), + new Document(92), + new Document(78) + }), + ["metadata"] = new Document(new Dictionary + { + ["version"] = new Document(1.5), + ["tags"] = new Document(new Document[] + { + new Document("test"), + new Document("production") + }) + }) + }) + } + } + } + } + }, + StopReason = new StopReason("tool_use") + }; + + // Act + var response = await client.GetResponseAsync(messages, options); + var json = JsonDocument.Parse(response.Text); + + // Assert nested object + var user = json.RootElement.GetProperty("user"); + Assert.Equal("John", user.GetProperty("name").GetString()); + Assert.Equal(30, user.GetProperty("age").GetInt32()); + Assert.True(user.GetProperty("active").GetBoolean()); + + // Assert array + var scores = json.RootElement.GetProperty("scores"); + Assert.Equal(JsonValueKind.Array, scores.ValueKind); + Assert.Equal(3, scores.GetArrayLength()); + Assert.Equal(85, scores[0].GetInt32()); + + // Assert nested object with array + var metadata = json.RootElement.GetProperty("metadata"); + Assert.Equal(1.5, metadata.GetProperty("version").GetDouble()); + var tags = metadata.GetProperty("tags"); + Assert.Equal(2, tags.GetArrayLength()); + Assert.Equal("test", tags[0].GetString()); + } + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Json_ModelReturnsNoToolUse_ThrowsInvalidOperationException() + { + // Arrange + var mock = new MockBedrockRuntime(); + var client = mock.AsIChatClient("claude-3"); + var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; + var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; + + // Mock returns text instead of tool use + mock.ResponseFactory = req => new ConverseResponse + { + Output = new ConverseOutput + { + Message = new Message + { + Role = ConversationRole.Assistant, + Content = new List + { + new ContentBlock { Text = "Regular text response" } + } + } + }, + StopReason = new StopReason("end_turn") + }; + + // Act & Assert + var ex = await Assert.ThrowsAsync(async () => + await client.GetResponseAsync(messages, options)); + + Assert.Contains("did not return structured output", ex.Message); + Assert.Contains("StopReason", ex.Message); + } + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Json_ModelReturnsWrongToolName_ThrowsInvalidOperationException() + { + // Arrange + var mock = new MockBedrockRuntime(); + var client = mock.AsIChatClient("claude-3"); + var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; + var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; + + // Mock returns tool use with wrong name + mock.ResponseFactory = req => new ConverseResponse + { + Output = new ConverseOutput + { + Message = new Message + { + Role = ConversationRole.Assistant, + Content = new List + { + new ContentBlock + { + ToolUse = new ToolUseBlock + { + ToolUseId = "test-id", + Name = "different_tool", // Wrong name! + Input = new Document(new Dictionary()) + } + } + } + } + }, + StopReason = new StopReason("tool_use") + }; + + // Act & Assert + var ex = await Assert.ThrowsAsync(async () => + await client.GetResponseAsync(messages, options)); + + Assert.Contains("did not return structured output", ex.Message); + } + + #endregion + + #region Document to JSON Conversion Tests + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task DocumentToJson_AllPrimitiveTypes_ConvertCorrectly() + { + // Arrange + var mock = new MockBedrockRuntime(); + var client = mock.AsIChatClient("claude-3"); + var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; + var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; + + mock.ResponseFactory = req => new ConverseResponse + { + Output = new ConverseOutput + { + Message = new Message + { + Role = ConversationRole.Assistant, + Content = new List + { + new ContentBlock + { + ToolUse = new ToolUseBlock + { + ToolUseId = "test-id", + Name = "generate_response", + Input = new Document(new Dictionary + { + ["stringVal"] = new Document("hello"), + ["intVal"] = new Document(42), + ["longVal"] = new Document(9876543210L), + ["doubleVal"] = new Document(3.14159), + ["boolTrue"] = new Document(true), + ["boolFalse"] = new Document(false) + }) + } + } + } + } + }, + StopReason = new StopReason("tool_use") + }; + + // Act + var response = await client.GetResponseAsync(messages, options); + var json = JsonDocument.Parse(response.Text); + + // Assert all types + Assert.Equal("hello", json.RootElement.GetProperty("stringVal").GetString()); + Assert.Equal(42, json.RootElement.GetProperty("intVal").GetInt32()); + Assert.Equal(9876543210L, json.RootElement.GetProperty("longVal").GetInt64()); + Assert.Equal(3.14159, json.RootElement.GetProperty("doubleVal").GetDouble(), precision: 5); + Assert.True(json.RootElement.GetProperty("boolTrue").GetBoolean()); + Assert.False(json.RootElement.GetProperty("boolFalse").GetBoolean()); + } + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task DocumentToJson_EmptyObjects_ConvertCorrectly() + { + // Arrange + var mock = new MockBedrockRuntime(); + var client = mock.AsIChatClient("claude-3"); + var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; + var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; + + mock.ResponseFactory = req => new ConverseResponse + { + Output = new ConverseOutput + { + Message = new Message + { + Role = ConversationRole.Assistant, + Content = new List + { + new ContentBlock + { + ToolUse = new ToolUseBlock + { + ToolUseId = "test-id", + Name = "generate_response", + Input = new Document(new Dictionary + { + ["emptyObject"] = new Document(new Dictionary()), + ["emptyArray"] = new Document(new Document[] { }) + }) + } + } + } + } + }, + StopReason = new StopReason("tool_use") + }; + + // Act + var response = await client.GetResponseAsync(messages, options); + var json = JsonDocument.Parse(response.Text); + + // Assert + Assert.Equal(JsonValueKind.Object, json.RootElement.GetProperty("emptyObject").ValueKind); + Assert.Empty(json.RootElement.GetProperty("emptyObject").EnumerateObject()); + + Assert.Equal(JsonValueKind.Array, json.RootElement.GetProperty("emptyArray").ValueKind); + Assert.Equal(0, json.RootElement.GetProperty("emptyArray").GetArrayLength()); + } + + #endregion + + #region Error Handling Tests + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Json_ValidationExceptionWithToolChoice_ThrowsNotSupportedException() + { + // Arrange + var mock = new MockBedrockRuntime(); + var client = mock.AsIChatClient("amazon.titan-text"); + var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; + var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; + + // Simulate model that doesn't support ToolChoice + mock.ExceptionToThrow = new AmazonBedrockRuntimeException("ValidationException: toolChoice is not supported for this model") + { + ErrorCode = "ValidationException" + }; + + // Act & Assert + var ex = await Assert.ThrowsAsync(async () => + await client.GetResponseAsync(messages, options)); + + Assert.Contains("does not support ResponseFormat", ex.Message); + Assert.Contains("ToolChoice support", ex.Message); + Assert.Contains("Claude 3+", ex.Message); + Assert.Contains("Mistral Large", ex.Message); + Assert.Contains("https://docs.aws.amazon.com/bedrock", ex.Message); + } + + [Theory] + [Trait("UnitTest", "BedrockRuntime")] + [InlineData("toolChoice")] + [InlineData("tool_choice")] + [InlineData("ToolChoice")] + public async Task ResponseFormat_Json_ValidationExceptionWithToolChoiceVariations_AllDetected(string keyword) + { + // Arrange + var mock = new MockBedrockRuntime(); + var client = mock.AsIChatClient("test-model"); + var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; + var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; + + mock.ExceptionToThrow = new AmazonBedrockRuntimeException($"ValidationException: {keyword} not supported") + { + ErrorCode = "ValidationException" + }; + + // Act & Assert + var ex = await Assert.ThrowsAsync(async () => + await client.GetResponseAsync(messages, options)); + + Assert.Contains("does not support ResponseFormat", ex.Message); + Assert.NotNull(ex.InnerException); + Assert.IsType(ex.InnerException); + } + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Json_OtherValidationException_RethrowsOriginal() + { + // Arrange + var mock = new MockBedrockRuntime(); + var client = mock.AsIChatClient("claude-3"); + var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; + var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; + + // Different validation error + mock.ExceptionToThrow = new AmazonBedrockRuntimeException("ValidationException: Invalid message format") + { + ErrorCode = "ValidationException" + }; + + // Act & Assert + var ex = await Assert.ThrowsAsync(async () => + await client.GetResponseAsync(messages, options)); + + Assert.Contains("Invalid message format", ex.Message); + } + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Json_NonValidationException_RethrowsOriginal() + { + // Arrange + var mock = new MockBedrockRuntime(); + var client = mock.AsIChatClient("claude-3"); + var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; + var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; + + mock.ExceptionToThrow = new AmazonBedrockRuntimeException("ThrottlingException: Rate limit exceeded") + { + ErrorCode = "ThrottlingException" + }; + + // Act & Assert + var ex = await Assert.ThrowsAsync(async () => + await client.GetResponseAsync(messages, options)); + + Assert.Equal("ThrottlingException", ex.ErrorCode); + } + + #endregion + + #region Streaming Tests + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Json_StreamingThrowsNotSupportedException() + { + // Arrange + var mock = new MockBedrockRuntime(); + var client = mock.AsIChatClient("claude-3"); + var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; + var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; + + // Act & Assert + var ex = await Assert.ThrowsAsync(async () => + { + await foreach (var update in client.GetStreamingResponseAsync(messages, options)) + { + Assert.Fail("Should not get here"); + } + }); + + Assert.Contains("ResponseFormat is not yet supported for streaming", ex.Message); + Assert.Contains("GetResponseAsync", ex.Message); + } + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Text_StreamingWorks() + { + // ResponseFormat.Text should NOT block streaming since it doesn't use synthetic tools + var mock = new MockBedrockRuntime(); + var client = mock.AsIChatClient("claude-3"); + var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; + var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Text }; + + // Should not throw - will throw NotImplementedException from our mock instead + await Assert.ThrowsAsync(async () => + { + await foreach (var update in client.GetStreamingResponseAsync(messages, options)) + { + } + }); + } + + #endregion + + #region Edge Cases and Integration + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Json_PreservesUsageMetadata() + { + // Arrange + var mock = new MockBedrockRuntime(); + var client = mock.AsIChatClient("claude-3"); + var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; + var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; + + mock.ResponseFactory = req => new ConverseResponse + { + Output = new ConverseOutput + { + Message = new Message + { + Role = ConversationRole.Assistant, + Content = new List + { + new ContentBlock + { + ToolUse = new ToolUseBlock + { + ToolUseId = "test-id", + Name = "generate_response", + Input = new Document(new Dictionary { ["result"] = new Document("test") }) + } + } + } + } + }, + StopReason = new StopReason("tool_use"), + Usage = new TokenUsage + { + InputTokens = 100, + OutputTokens = 50, + TotalTokens = 150 + } + }; + + // Act + var response = await client.GetResponseAsync(messages, options); + + // Assert + Assert.NotNull(response.Usage); + Assert.Equal(100, response.Usage.InputTokenCount); + Assert.Equal(50, response.Usage.OutputTokenCount); + Assert.Equal(150, response.Usage.TotalTokenCount); + } + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Json_PreservesFinishReason() + { + // Arrange + var mock = new MockBedrockRuntime(); + var client = mock.AsIChatClient("claude-3"); + var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; + var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; + + mock.ResponseFactory = req => new ConverseResponse + { + Output = new ConverseOutput + { + Message = new Message + { + Role = ConversationRole.Assistant, + Content = new List + { + new ContentBlock + { + ToolUse = new ToolUseBlock + { + ToolUseId = "test-id", + Name = "generate_response", + Input = new Document(new Dictionary()) + } + } + } + } + }, + StopReason = new StopReason("tool_use") + }; + + // Act + var response = await client.GetResponseAsync(messages, options); + + // Assert + Assert.NotNull(response.FinishReason); + Assert.Equal(ChatFinishReason.ToolCalls, response.FinishReason); + } + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Json_EmptyInput_HandlesGracefully() + { + // Arrange + var mock = new MockBedrockRuntime(); + var client = mock.AsIChatClient("claude-3"); + var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; + var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; + + mock.ResponseFactory = req => new ConverseResponse + { + Output = new ConverseOutput + { + Message = new Message + { + Role = ConversationRole.Assistant, + Content = new List + { + new ContentBlock + { + ToolUse = new ToolUseBlock + { + ToolUseId = "test-id", + Name = "generate_response", + Input = new Document(new Dictionary()) // Empty object + } + } + } + } + }, + StopReason = new StopReason("tool_use") + }; + + // Act + var response = await client.GetResponseAsync(messages, options); + + // Assert + Assert.NotNull(response.Text); + var json = JsonDocument.Parse(response.Text); + Assert.Equal(JsonValueKind.Object, json.RootElement.ValueKind); + Assert.Empty(json.RootElement.EnumerateObject()); + } + + #endregion } From 58e17cd3ced3b63221aacd56d55c6efec4d63841 Mon Sep 17 00:00:00 2001 From: adaines Date: Thu, 6 Nov 2025 15:41:54 -0500 Subject: [PATCH 2/7] removed redundant tests --- .../BedrockChatClientTests.cs | 673 +----------------- 1 file changed, 1 insertion(+), 672 deletions(-) diff --git a/extensions/test/BedrockMEAITests/BedrockChatClientTests.cs b/extensions/test/BedrockMEAITests/BedrockChatClientTests.cs index 58473a8438a3..b599c643bf3d 100644 --- a/extensions/test/BedrockMEAITests/BedrockChatClientTests.cs +++ b/extensions/test/BedrockMEAITests/BedrockChatClientTests.cs @@ -142,53 +142,7 @@ public void AsIChatClient_GetService() #endregion - #region Synthetic Tool Creation Tests - - [Fact] - [Trait("UnitTest", "BedrockRuntime")] - public async Task ResponseFormat_Json_WithoutSchema_CreatesSyntheticToolWithGenericObjectSchema() - { - // Arrange - var mock = new MockBedrockRuntime(); - var client = mock.AsIChatClient("claude-3"); - var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; - var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; - - // Act - try - { - await client.GetResponseAsync(messages, options); - } - catch - { - // We're testing request creation, not response processing - } - - // Assert - Verify synthetic tool was created correctly - Assert.NotNull(mock.CapturedRequest); - Assert.NotNull(mock.CapturedRequest.ToolConfig); - Assert.NotNull(mock.CapturedRequest.ToolConfig.Tools); - Assert.Single(mock.CapturedRequest.ToolConfig.Tools); - - var tool = mock.CapturedRequest.ToolConfig.Tools[0]; - Assert.NotNull(tool.ToolSpec); - Assert.Equal("generate_response", tool.ToolSpec.Name); - Assert.Equal("Generate response in specified format", tool.ToolSpec.Description); - - // Verify schema is generic object schema - var schema = tool.ToolSpec.InputSchema.Json; - Assert.True(schema.IsDictionary()); - var schemaDict = schema.AsDictionary(); - Assert.True(schemaDict.ContainsKey("type")); - Assert.Equal("object", schemaDict["type"].AsString()); - Assert.True(schemaDict.ContainsKey("additionalProperties")); - Assert.True(schemaDict["additionalProperties"].AsBool()); - - // Verify ToolChoice forces the synthetic tool - Assert.NotNull(mock.CapturedRequest.ToolConfig.ToolChoice); - Assert.NotNull(mock.CapturedRequest.ToolConfig.ToolChoice.Tool); - Assert.Equal("generate_response", mock.CapturedRequest.ToolConfig.ToolChoice.Tool.Name); - } + #region ResponseFormat Tests [Fact] [Trait("UnitTest", "BedrockRuntime")] @@ -252,74 +206,6 @@ public async Task ResponseFormat_Json_WithSchema_CreatesSyntheticToolWithCorrect Assert.Equal("name", required[0].AsString()); } - [Fact] - [Trait("UnitTest", "BedrockRuntime")] - public async Task ResponseFormat_Json_WithTools_ThrowsArgumentException() - { - // Arrange - var mock = new MockBedrockRuntime(); - var client = mock.AsIChatClient("claude-3"); - var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; - - var toolSchema = JsonDocument.Parse("""{"type": "object"}""").RootElement; - var options = new ChatOptions - { - ResponseFormat = ChatResponseFormat.Json, - Tools = new List - { - new TestAIFunction("test_tool", "Test", toolSchema) - } - }; - - // Act & Assert - var ex = await Assert.ThrowsAsync(async () => - await client.GetResponseAsync(messages, options)); - - Assert.Contains("ResponseFormat cannot be used with Tools", ex.Message); - Assert.Contains("Bedrock's tool mechanism", ex.Message); - } - - [Fact] - [Trait("UnitTest", "BedrockRuntime")] - public async Task ResponseFormat_Text_DoesNotCreateSyntheticTool() - { - // Arrange - var mock = new MockBedrockRuntime(); - var client = mock.AsIChatClient("claude-3"); - var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; - var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Text }; - - // Act - await client.GetResponseAsync(messages, options); - - // Assert - No synthetic tool should be created for Text format - Assert.True(mock.CapturedRequest.ToolConfig == null || - mock.CapturedRequest.ToolConfig.Tools == null || - mock.CapturedRequest.ToolConfig.Tools.Count == 0); - } - - [Fact] - [Trait("UnitTest", "BedrockRuntime")] - public async Task ResponseFormat_NotSet_DoesNotCreateSyntheticTool() - { - // Arrange - var mock = new MockBedrockRuntime(); - var client = mock.AsIChatClient("claude-3"); - var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; - - // Act - await client.GetResponseAsync(messages); - - // Assert - Assert.True(mock.CapturedRequest.ToolConfig == null || - mock.CapturedRequest.ToolConfig.Tools == null || - mock.CapturedRequest.ToolConfig.Tools.Count == 0); - } - - #endregion - - #region Response Extraction Tests - [Fact] [Trait("UnitTest", "BedrockRuntime")] public async Task ResponseFormat_Json_ModelReturnsToolUse_ExtractsJsonCorrectly() @@ -375,562 +261,5 @@ public async Task ResponseFormat_Json_ModelReturnsToolUse_ExtractsJsonCorrectly( Assert.Equal("sunny", json.RootElement.GetProperty("conditions").GetString()); } - [Fact] - [Trait("UnitTest", "BedrockRuntime")] - public async Task ResponseFormat_Json_ModelReturnsComplexNestedData_ExtractsCorrectly() - { - // Arrange - var mock = new MockBedrockRuntime(); - var client = mock.AsIChatClient("claude-3"); - var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; - var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; - - // Setup complex nested structure - mock.ResponseFactory = req => new ConverseResponse - { - Output = new ConverseOutput - { - Message = new Message - { - Role = ConversationRole.Assistant, - Content = new List - { - new ContentBlock - { - ToolUse = new ToolUseBlock - { - ToolUseId = "test-id", - Name = "generate_response", - Input = new Document(new Dictionary - { - ["user"] = new Document(new Dictionary - { - ["name"] = new Document("John"), - ["age"] = new Document(30), - ["active"] = new Document(true) - }), - ["scores"] = new Document(new Document[] - { - new Document(85), - new Document(92), - new Document(78) - }), - ["metadata"] = new Document(new Dictionary - { - ["version"] = new Document(1.5), - ["tags"] = new Document(new Document[] - { - new Document("test"), - new Document("production") - }) - }) - }) - } - } - } - } - }, - StopReason = new StopReason("tool_use") - }; - - // Act - var response = await client.GetResponseAsync(messages, options); - var json = JsonDocument.Parse(response.Text); - - // Assert nested object - var user = json.RootElement.GetProperty("user"); - Assert.Equal("John", user.GetProperty("name").GetString()); - Assert.Equal(30, user.GetProperty("age").GetInt32()); - Assert.True(user.GetProperty("active").GetBoolean()); - - // Assert array - var scores = json.RootElement.GetProperty("scores"); - Assert.Equal(JsonValueKind.Array, scores.ValueKind); - Assert.Equal(3, scores.GetArrayLength()); - Assert.Equal(85, scores[0].GetInt32()); - - // Assert nested object with array - var metadata = json.RootElement.GetProperty("metadata"); - Assert.Equal(1.5, metadata.GetProperty("version").GetDouble()); - var tags = metadata.GetProperty("tags"); - Assert.Equal(2, tags.GetArrayLength()); - Assert.Equal("test", tags[0].GetString()); - } - - [Fact] - [Trait("UnitTest", "BedrockRuntime")] - public async Task ResponseFormat_Json_ModelReturnsNoToolUse_ThrowsInvalidOperationException() - { - // Arrange - var mock = new MockBedrockRuntime(); - var client = mock.AsIChatClient("claude-3"); - var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; - var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; - - // Mock returns text instead of tool use - mock.ResponseFactory = req => new ConverseResponse - { - Output = new ConverseOutput - { - Message = new Message - { - Role = ConversationRole.Assistant, - Content = new List - { - new ContentBlock { Text = "Regular text response" } - } - } - }, - StopReason = new StopReason("end_turn") - }; - - // Act & Assert - var ex = await Assert.ThrowsAsync(async () => - await client.GetResponseAsync(messages, options)); - - Assert.Contains("did not return structured output", ex.Message); - Assert.Contains("StopReason", ex.Message); - } - - [Fact] - [Trait("UnitTest", "BedrockRuntime")] - public async Task ResponseFormat_Json_ModelReturnsWrongToolName_ThrowsInvalidOperationException() - { - // Arrange - var mock = new MockBedrockRuntime(); - var client = mock.AsIChatClient("claude-3"); - var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; - var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; - - // Mock returns tool use with wrong name - mock.ResponseFactory = req => new ConverseResponse - { - Output = new ConverseOutput - { - Message = new Message - { - Role = ConversationRole.Assistant, - Content = new List - { - new ContentBlock - { - ToolUse = new ToolUseBlock - { - ToolUseId = "test-id", - Name = "different_tool", // Wrong name! - Input = new Document(new Dictionary()) - } - } - } - } - }, - StopReason = new StopReason("tool_use") - }; - - // Act & Assert - var ex = await Assert.ThrowsAsync(async () => - await client.GetResponseAsync(messages, options)); - - Assert.Contains("did not return structured output", ex.Message); - } - - #endregion - - #region Document to JSON Conversion Tests - - [Fact] - [Trait("UnitTest", "BedrockRuntime")] - public async Task DocumentToJson_AllPrimitiveTypes_ConvertCorrectly() - { - // Arrange - var mock = new MockBedrockRuntime(); - var client = mock.AsIChatClient("claude-3"); - var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; - var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; - - mock.ResponseFactory = req => new ConverseResponse - { - Output = new ConverseOutput - { - Message = new Message - { - Role = ConversationRole.Assistant, - Content = new List - { - new ContentBlock - { - ToolUse = new ToolUseBlock - { - ToolUseId = "test-id", - Name = "generate_response", - Input = new Document(new Dictionary - { - ["stringVal"] = new Document("hello"), - ["intVal"] = new Document(42), - ["longVal"] = new Document(9876543210L), - ["doubleVal"] = new Document(3.14159), - ["boolTrue"] = new Document(true), - ["boolFalse"] = new Document(false) - }) - } - } - } - } - }, - StopReason = new StopReason("tool_use") - }; - - // Act - var response = await client.GetResponseAsync(messages, options); - var json = JsonDocument.Parse(response.Text); - - // Assert all types - Assert.Equal("hello", json.RootElement.GetProperty("stringVal").GetString()); - Assert.Equal(42, json.RootElement.GetProperty("intVal").GetInt32()); - Assert.Equal(9876543210L, json.RootElement.GetProperty("longVal").GetInt64()); - Assert.Equal(3.14159, json.RootElement.GetProperty("doubleVal").GetDouble(), precision: 5); - Assert.True(json.RootElement.GetProperty("boolTrue").GetBoolean()); - Assert.False(json.RootElement.GetProperty("boolFalse").GetBoolean()); - } - - [Fact] - [Trait("UnitTest", "BedrockRuntime")] - public async Task DocumentToJson_EmptyObjects_ConvertCorrectly() - { - // Arrange - var mock = new MockBedrockRuntime(); - var client = mock.AsIChatClient("claude-3"); - var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; - var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; - - mock.ResponseFactory = req => new ConverseResponse - { - Output = new ConverseOutput - { - Message = new Message - { - Role = ConversationRole.Assistant, - Content = new List - { - new ContentBlock - { - ToolUse = new ToolUseBlock - { - ToolUseId = "test-id", - Name = "generate_response", - Input = new Document(new Dictionary - { - ["emptyObject"] = new Document(new Dictionary()), - ["emptyArray"] = new Document(new Document[] { }) - }) - } - } - } - } - }, - StopReason = new StopReason("tool_use") - }; - - // Act - var response = await client.GetResponseAsync(messages, options); - var json = JsonDocument.Parse(response.Text); - - // Assert - Assert.Equal(JsonValueKind.Object, json.RootElement.GetProperty("emptyObject").ValueKind); - Assert.Empty(json.RootElement.GetProperty("emptyObject").EnumerateObject()); - - Assert.Equal(JsonValueKind.Array, json.RootElement.GetProperty("emptyArray").ValueKind); - Assert.Equal(0, json.RootElement.GetProperty("emptyArray").GetArrayLength()); - } - - #endregion - - #region Error Handling Tests - - [Fact] - [Trait("UnitTest", "BedrockRuntime")] - public async Task ResponseFormat_Json_ValidationExceptionWithToolChoice_ThrowsNotSupportedException() - { - // Arrange - var mock = new MockBedrockRuntime(); - var client = mock.AsIChatClient("amazon.titan-text"); - var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; - var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; - - // Simulate model that doesn't support ToolChoice - mock.ExceptionToThrow = new AmazonBedrockRuntimeException("ValidationException: toolChoice is not supported for this model") - { - ErrorCode = "ValidationException" - }; - - // Act & Assert - var ex = await Assert.ThrowsAsync(async () => - await client.GetResponseAsync(messages, options)); - - Assert.Contains("does not support ResponseFormat", ex.Message); - Assert.Contains("ToolChoice support", ex.Message); - Assert.Contains("Claude 3+", ex.Message); - Assert.Contains("Mistral Large", ex.Message); - Assert.Contains("https://docs.aws.amazon.com/bedrock", ex.Message); - } - - [Theory] - [Trait("UnitTest", "BedrockRuntime")] - [InlineData("toolChoice")] - [InlineData("tool_choice")] - [InlineData("ToolChoice")] - public async Task ResponseFormat_Json_ValidationExceptionWithToolChoiceVariations_AllDetected(string keyword) - { - // Arrange - var mock = new MockBedrockRuntime(); - var client = mock.AsIChatClient("test-model"); - var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; - var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; - - mock.ExceptionToThrow = new AmazonBedrockRuntimeException($"ValidationException: {keyword} not supported") - { - ErrorCode = "ValidationException" - }; - - // Act & Assert - var ex = await Assert.ThrowsAsync(async () => - await client.GetResponseAsync(messages, options)); - - Assert.Contains("does not support ResponseFormat", ex.Message); - Assert.NotNull(ex.InnerException); - Assert.IsType(ex.InnerException); - } - - [Fact] - [Trait("UnitTest", "BedrockRuntime")] - public async Task ResponseFormat_Json_OtherValidationException_RethrowsOriginal() - { - // Arrange - var mock = new MockBedrockRuntime(); - var client = mock.AsIChatClient("claude-3"); - var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; - var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; - - // Different validation error - mock.ExceptionToThrow = new AmazonBedrockRuntimeException("ValidationException: Invalid message format") - { - ErrorCode = "ValidationException" - }; - - // Act & Assert - var ex = await Assert.ThrowsAsync(async () => - await client.GetResponseAsync(messages, options)); - - Assert.Contains("Invalid message format", ex.Message); - } - - [Fact] - [Trait("UnitTest", "BedrockRuntime")] - public async Task ResponseFormat_Json_NonValidationException_RethrowsOriginal() - { - // Arrange - var mock = new MockBedrockRuntime(); - var client = mock.AsIChatClient("claude-3"); - var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; - var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; - - mock.ExceptionToThrow = new AmazonBedrockRuntimeException("ThrottlingException: Rate limit exceeded") - { - ErrorCode = "ThrottlingException" - }; - - // Act & Assert - var ex = await Assert.ThrowsAsync(async () => - await client.GetResponseAsync(messages, options)); - - Assert.Equal("ThrottlingException", ex.ErrorCode); - } - - #endregion - - #region Streaming Tests - - [Fact] - [Trait("UnitTest", "BedrockRuntime")] - public async Task ResponseFormat_Json_StreamingThrowsNotSupportedException() - { - // Arrange - var mock = new MockBedrockRuntime(); - var client = mock.AsIChatClient("claude-3"); - var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; - var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; - - // Act & Assert - var ex = await Assert.ThrowsAsync(async () => - { - await foreach (var update in client.GetStreamingResponseAsync(messages, options)) - { - Assert.Fail("Should not get here"); - } - }); - - Assert.Contains("ResponseFormat is not yet supported for streaming", ex.Message); - Assert.Contains("GetResponseAsync", ex.Message); - } - - [Fact] - [Trait("UnitTest", "BedrockRuntime")] - public async Task ResponseFormat_Text_StreamingWorks() - { - // ResponseFormat.Text should NOT block streaming since it doesn't use synthetic tools - var mock = new MockBedrockRuntime(); - var client = mock.AsIChatClient("claude-3"); - var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; - var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Text }; - - // Should not throw - will throw NotImplementedException from our mock instead - await Assert.ThrowsAsync(async () => - { - await foreach (var update in client.GetStreamingResponseAsync(messages, options)) - { - } - }); - } - - #endregion - - #region Edge Cases and Integration - - [Fact] - [Trait("UnitTest", "BedrockRuntime")] - public async Task ResponseFormat_Json_PreservesUsageMetadata() - { - // Arrange - var mock = new MockBedrockRuntime(); - var client = mock.AsIChatClient("claude-3"); - var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; - var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; - - mock.ResponseFactory = req => new ConverseResponse - { - Output = new ConverseOutput - { - Message = new Message - { - Role = ConversationRole.Assistant, - Content = new List - { - new ContentBlock - { - ToolUse = new ToolUseBlock - { - ToolUseId = "test-id", - Name = "generate_response", - Input = new Document(new Dictionary { ["result"] = new Document("test") }) - } - } - } - } - }, - StopReason = new StopReason("tool_use"), - Usage = new TokenUsage - { - InputTokens = 100, - OutputTokens = 50, - TotalTokens = 150 - } - }; - - // Act - var response = await client.GetResponseAsync(messages, options); - - // Assert - Assert.NotNull(response.Usage); - Assert.Equal(100, response.Usage.InputTokenCount); - Assert.Equal(50, response.Usage.OutputTokenCount); - Assert.Equal(150, response.Usage.TotalTokenCount); - } - - [Fact] - [Trait("UnitTest", "BedrockRuntime")] - public async Task ResponseFormat_Json_PreservesFinishReason() - { - // Arrange - var mock = new MockBedrockRuntime(); - var client = mock.AsIChatClient("claude-3"); - var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; - var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; - - mock.ResponseFactory = req => new ConverseResponse - { - Output = new ConverseOutput - { - Message = new Message - { - Role = ConversationRole.Assistant, - Content = new List - { - new ContentBlock - { - ToolUse = new ToolUseBlock - { - ToolUseId = "test-id", - Name = "generate_response", - Input = new Document(new Dictionary()) - } - } - } - } - }, - StopReason = new StopReason("tool_use") - }; - - // Act - var response = await client.GetResponseAsync(messages, options); - - // Assert - Assert.NotNull(response.FinishReason); - Assert.Equal(ChatFinishReason.ToolCalls, response.FinishReason); - } - - [Fact] - [Trait("UnitTest", "BedrockRuntime")] - public async Task ResponseFormat_Json_EmptyInput_HandlesGracefully() - { - // Arrange - var mock = new MockBedrockRuntime(); - var client = mock.AsIChatClient("claude-3"); - var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; - var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; - - mock.ResponseFactory = req => new ConverseResponse - { - Output = new ConverseOutput - { - Message = new Message - { - Role = ConversationRole.Assistant, - Content = new List - { - new ContentBlock - { - ToolUse = new ToolUseBlock - { - ToolUseId = "test-id", - Name = "generate_response", - Input = new Document(new Dictionary()) // Empty object - } - } - } - } - }, - StopReason = new StopReason("tool_use") - }; - - // Act - var response = await client.GetResponseAsync(messages, options); - - // Assert - Assert.NotNull(response.Text); - var json = JsonDocument.Parse(response.Text); - Assert.Equal(JsonValueKind.Object, json.RootElement.ValueKind); - Assert.Empty(json.RootElement.EnumerateObject()); - } - #endregion } From cf26be9096791869c20329a64f4dea71d417675f Mon Sep 17 00:00:00 2001 From: adaines Date: Tue, 18 Nov 2025 13:04:38 -0500 Subject: [PATCH 3/7] Address PR #4113 feedback: use DocumentMarshaller, clarify exception handling, restore docs --- .../BedrockChatClient.cs | 96 ++++--------------- 1 file changed, 21 insertions(+), 75 deletions(-) diff --git a/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/BedrockChatClient.cs b/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/BedrockChatClient.cs index 1c95ce8a5402..b9015c260a0d 100644 --- a/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/BedrockChatClient.cs +++ b/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/BedrockChatClient.cs @@ -41,8 +41,6 @@ internal sealed partial class BedrockChatClient : IChatClient private const string ResponseFormatToolName = "generate_response"; /// The description used for the synthetic tool that enforces response format. private const string ResponseFormatToolDescription = "Generate response in specified format"; - /// Maximum nesting depth for Document to JSON conversion to prevent stack overflow. - private const int MaxDocumentNestingDepth = 100; /// The wrapped instance. private readonly IAmazonBedrockRuntime _runtime; @@ -51,7 +49,11 @@ internal sealed partial class BedrockChatClient : IChatClient /// Metadata describing the chat client. private readonly ChatClientMetadata _metadata; - /// Initializes a new instance of the class. + /// + /// Initializes a new instance of the class. + /// + /// The instance to wrap. + /// Model ID to use as the default when no model ID is specified in a request. public BedrockChatClient(IAmazonBedrockRuntime runtime, string? defaultModelId) { Debug.Assert(runtime is not null); @@ -68,6 +70,12 @@ public void Dispose() } /// + /// + /// When is specified, the model must support + /// the ToolChoice feature. Models without this support will throw . + /// If the model fails to return the expected structured output, + /// is thrown. + /// public async Task GetResponseAsync( IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) { @@ -84,7 +92,6 @@ public async Task GetResponseAsync( request.InferenceConfig = CreateInferenceConfiguration(request.InferenceConfig, options); request.AdditionalModelRequestFields = CreateAdditionalModelRequestFields(request.AdditionalModelRequestFields, options); - // Execute the request with proper error handling for ResponseFormat scenarios ConverseResponse response; try { @@ -92,16 +99,10 @@ public async Task GetResponseAsync( } catch (AmazonBedrockRuntimeException ex) when (options?.ResponseFormat is ChatResponseFormatJson) { - // Check if this is a ToolChoice validation error (model doesn't support it) - bool isToolChoiceNotSupported = - ex.ErrorCode == "ValidationException" && - (ex.Message.IndexOf("toolChoice", StringComparison.OrdinalIgnoreCase) >= 0 || - ex.Message.IndexOf("tool_choice", StringComparison.OrdinalIgnoreCase) >= 0 || - ex.Message.IndexOf("ToolChoice", StringComparison.OrdinalIgnoreCase) >= 0); - - if (isToolChoiceNotSupported) + // Detect unsupported model: ValidationException mentioning "toolChoice" + if (ex.ErrorCode == "ValidationException" && + ex.Message.IndexOf("toolchoice", StringComparison.OrdinalIgnoreCase) >= 0) { - // Provide a more helpful error message when ToolChoice fails due to model limitations throw new NotSupportedException( $"The model '{request.ModelId}' does not support ResponseFormat. " + $"ResponseFormat requires ToolChoice support, which is only available in Claude 3+ and Mistral Large models. " + @@ -109,7 +110,6 @@ public async Task GetResponseAsync( ex); } - // Re-throw other exceptions as-is throw; } @@ -147,7 +147,7 @@ public async Task GetResponseAsync( } else { - // User requested structured output but didn't get it - this is a contract violation + // Model succeeded but did not return expected structured output var errorMessage = string.Format( "ResponseFormat was specified but model did not return expected tool use. ModelId: {0}, StopReason: {1}", request.ModelId, @@ -155,7 +155,6 @@ public async Task GetResponseAsync( DefaultLogger.Error(new InvalidOperationException(errorMessage), errorMessage); - // Always throw when ResponseFormat was requested but not fulfilled throw new InvalidOperationException( $"Model '{request.ModelId}' did not return structured output as requested. " + $"This may indicate the model refused to follow the tool use instruction, " + @@ -1032,73 +1031,20 @@ private static Document ToDocument(JsonElement json) return null; } - /// Converts a to a JSON string. + /// + /// Converts a to a JSON string using the SDK's standard DocumentMarshaller. + /// Note: Document is a struct (value type), so circular references are structurally impossible. + /// private static string DocumentToJsonString(Document document) { using var stream = new MemoryStream(); using (var writer = new Utf8JsonWriter(stream, new JsonWriterOptions { Indented = false })) { - WriteDocumentAsJson(writer, document); - } // Explicit scope to ensure writer is flushed before reading buffer - + Amazon.Runtime.Documents.Internal.Transform.DocumentMarshaller.Instance.Write(writer, document); + } return Encoding.UTF8.GetString(stream.ToArray()); } - /// Recursively writes a as JSON. - private static void WriteDocumentAsJson(Utf8JsonWriter writer, Document document, int depth = 0) - { - // Check depth to prevent stack overflow from deeply nested or circular structures - if (depth > MaxDocumentNestingDepth) - { - throw new InvalidOperationException( - $"Document nesting depth exceeds maximum of {MaxDocumentNestingDepth}. " + - $"This may indicate a circular reference or excessively nested data structure."); - } - - if (document.IsBool()) - { - writer.WriteBooleanValue(document.AsBool()); - } - else if (document.IsInt()) - { - writer.WriteNumberValue(document.AsInt()); - } - else if (document.IsLong()) - { - writer.WriteNumberValue(document.AsLong()); - } - else if (document.IsDouble()) - { - writer.WriteNumberValue(document.AsDouble()); - } - else if (document.IsString()) - { - writer.WriteStringValue(document.AsString()); - } - else if (document.IsDictionary()) - { - writer.WriteStartObject(); - foreach (var kvp in document.AsDictionary()) - { - writer.WritePropertyName(kvp.Key); - WriteDocumentAsJson(writer, kvp.Value, depth + 1); - } - writer.WriteEndObject(); - } - else if (document.IsList()) - { - writer.WriteStartArray(); - foreach (var item in document.AsList()) - { - WriteDocumentAsJson(writer, item, depth + 1); - } - writer.WriteEndArray(); - } - else - { - writer.WriteNullValue(); - } - } /// Creates an from the specified options. private static InferenceConfiguration CreateInferenceConfiguration(InferenceConfiguration config, ChatOptions? options) From 062070c2ce224c9ffe33909947518c49e2b5b968 Mon Sep 17 00:00:00 2001 From: adaines Date: Wed, 19 Nov 2025 09:46:56 -0500 Subject: [PATCH 4/7] Fix error handling issues and add critical error scenario test coverage per PR feedback --- .../BedrockChatClient.cs | 17 +- .../BedrockChatClientTests.cs | 1142 +++++++++++++++-- 2 files changed, 1033 insertions(+), 126 deletions(-) diff --git a/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/BedrockChatClient.cs b/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/BedrockChatClient.cs index b9015c260a0d..3c828ca4b27b 100644 --- a/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/BedrockChatClient.cs +++ b/extensions/src/AWSSDK.Extensions.Bedrock.MEAI/BedrockChatClient.cs @@ -21,6 +21,7 @@ using System.Buffers; using System.Collections.Generic; using System.Diagnostics; +using System.Globalization; using System.IO; using System.Linq; using System.Runtime.CompilerServices; @@ -97,11 +98,14 @@ public async Task GetResponseAsync( { response = await _runtime.ConverseAsync(request, cancellationToken).ConfigureAwait(false); } + // Transforms ValidationException to NotSupportedException when error message indicates model lacks tool use support (required for ResponseFormat). + // This detection relies on error message text which may change in future Bedrock API versions. catch (AmazonBedrockRuntimeException ex) when (options?.ResponseFormat is ChatResponseFormatJson) { - // Detect unsupported model: ValidationException mentioning "toolChoice" + // Detect unsupported model: ValidationException with specific tool support error messages if (ex.ErrorCode == "ValidationException" && - ex.Message.IndexOf("toolchoice", StringComparison.OrdinalIgnoreCase) >= 0) + (ex.Message.IndexOf("toolChoice is not supported by this model", StringComparison.OrdinalIgnoreCase) >= 0 || + ex.Message.IndexOf("This model doesn't support tool use", StringComparison.OrdinalIgnoreCase) >= 0)) { throw new NotSupportedException( $"The model '{request.ModelId}' does not support ResponseFormat. " + @@ -148,18 +152,11 @@ public async Task GetResponseAsync( else { // Model succeeded but did not return expected structured output - var errorMessage = string.Format( - "ResponseFormat was specified but model did not return expected tool use. ModelId: {0}, StopReason: {1}", - request.ModelId, - response.StopReason?.Value ?? "unknown"); - - DefaultLogger.Error(new InvalidOperationException(errorMessage), errorMessage); - throw new InvalidOperationException( $"Model '{request.ModelId}' did not return structured output as requested. " + $"This may indicate the model refused to follow the tool use instruction, " + $"the schema was too complex, or the prompt conflicted with the requirement. " + - $"StopReason: {response.StopReason?.Value ?? "unknown"}"); + $"StopReason: {response.StopReason?.Value ?? "unknown"}."); } } diff --git a/extensions/test/BedrockMEAITests/BedrockChatClientTests.cs b/extensions/test/BedrockMEAITests/BedrockChatClientTests.cs index b599c643bf3d..4b180da27c7c 100644 --- a/extensions/test/BedrockMEAITests/BedrockChatClientTests.cs +++ b/extensions/test/BedrockMEAITests/BedrockChatClientTests.cs @@ -1,9 +1,18 @@ using Amazon.BedrockRuntime.Model; +using Amazon.Runtime; using Amazon.Runtime.Documents; +using Amazon.Runtime.Internal; +using Amazon.Runtime.Internal.Transform; using Microsoft.Extensions.AI; +using Moq; using System; using System.Collections.Generic; +using System.IO; using System.Linq; +using System.Net; +using System.Net.Http; +using System.Reflection; +using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -11,82 +20,6 @@ namespace Amazon.BedrockRuntime; -// Mock implementation to capture requests and control responses -internal sealed class MockBedrockRuntime : IAmazonBedrockRuntime -{ - public ConverseRequest CapturedRequest { get; private set; } - public ConverseStreamRequest CapturedStreamRequest { get; private set; } - public Func ResponseFactory { get; set; } - public Exception ExceptionToThrow { get; set; } - - public Task ConverseAsync(ConverseRequest request, CancellationToken cancellationToken = default) - { - CapturedRequest = request; - - if (ExceptionToThrow != null) - { - throw ExceptionToThrow; - } - - if (ResponseFactory != null) - { - return Task.FromResult(ResponseFactory(request)); - } - - // Default response - return Task.FromResult(new ConverseResponse - { - Output = new ConverseOutput - { - Message = new Message - { - Role = ConversationRole.Assistant, - Content = new List - { - new ContentBlock { Text = "Default response" } - } - } - }, - StopReason = new StopReason("end_turn") - }); - } - - public Task ConverseStreamAsync(ConverseStreamRequest request, CancellationToken cancellationToken = default) - { - CapturedStreamRequest = request; - throw new NotImplementedException("Stream testing not implemented in this mock"); - } - - public void Dispose() { } - - // Unused interface members - all throw NotImplementedException - public IBedrockRuntimePaginatorFactory Paginators => throw new NotImplementedException(); - public Amazon.Runtime.IClientConfig Config => throw new NotImplementedException(); - - // Sync methods - public ApplyGuardrailResponse ApplyGuardrail(ApplyGuardrailRequest request) => throw new NotImplementedException(); - public ConverseResponse Converse(ConverseRequest request) => throw new NotImplementedException(); - public ConverseStreamResponse ConverseStream(ConverseStreamRequest request) => throw new NotImplementedException(); - public CountTokensResponse CountTokens(CountTokensRequest request) => throw new NotImplementedException(); - public GetAsyncInvokeResponse GetAsyncInvoke(GetAsyncInvokeRequest request) => throw new NotImplementedException(); - public InvokeModelResponse InvokeModel(InvokeModelRequest request) => throw new NotImplementedException(); - public InvokeModelWithResponseStreamResponse InvokeModelWithResponseStream(InvokeModelWithResponseStreamRequest request) => throw new NotImplementedException(); - public ListAsyncInvokesResponse ListAsyncInvokes(ListAsyncInvokesRequest request) => throw new NotImplementedException(); - public StartAsyncInvokeResponse StartAsyncInvoke(StartAsyncInvokeRequest request) => throw new NotImplementedException(); - - // Async methods - public Task ApplyGuardrailAsync(ApplyGuardrailRequest request, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public Task CountTokensAsync(CountTokensRequest request, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public Task GetAsyncInvokeAsync(GetAsyncInvokeRequest request, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public Task InvokeModelAsync(InvokeModelRequest request, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public Task InvokeModelWithResponseStreamAsync(InvokeModelWithResponseStreamRequest request, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public Task ListAsyncInvokesAsync(ListAsyncInvokesRequest request, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - public Task StartAsyncInvokeAsync(StartAsyncInvokeRequest request, CancellationToken cancellationToken = default) => throw new NotImplementedException(); - - // Endpoint determination - public Amazon.Runtime.Endpoints.Endpoint DetermineServiceOperationEndpoint(Amazon.Runtime.AmazonWebServiceRequest request) => throw new NotImplementedException(); -} - // Simple test implementation of AIFunctionDeclaration internal sealed class TestAIFunction : AIFunctionDeclaration { @@ -119,8 +52,8 @@ public void AsIChatClient_InvalidArguments_Throws() [InlineData("claude")] public void AsIChatClient_ReturnsInstance(string modelId) { - var mock = new MockBedrockRuntime(); - IChatClient client = mock.AsIChatClient(modelId); + var mockRuntime = new Mock(); + IChatClient client = mockRuntime.Object.AsIChatClient(modelId); Assert.NotNull(client); Assert.Equal("aws.bedrock", client.GetService()?.ProviderName); @@ -131,10 +64,10 @@ public void AsIChatClient_ReturnsInstance(string modelId) [Trait("UnitTest", "BedrockRuntime")] public void AsIChatClient_GetService() { - var mock = new MockBedrockRuntime(); - IChatClient client = mock.AsIChatClient(); + var mockRuntime = new Mock(); + IChatClient client = mockRuntime.Object.AsIChatClient(); - Assert.Same(mock, client.GetService()); + Assert.Same(mockRuntime.Object, client.GetService()); Assert.Same(client, client.GetService()); Assert.Null(client.GetService()); Assert.Null(client.GetService("key")); @@ -149,8 +82,41 @@ public void AsIChatClient_GetService() public async Task ResponseFormat_Json_WithSchema_CreatesSyntheticToolWithCorrectSchema() { // Arrange - var mock = new MockBedrockRuntime(); - var client = mock.AsIChatClient("claude-3"); + var mockRuntime = new Mock(); + ConverseRequest capturedRequest = null; + + mockRuntime + .Setup(x => x.ConverseAsync(It.IsAny(), It.IsAny())) + .Callback((req, ct) => capturedRequest = req) + .ReturnsAsync(new ConverseResponse + { + Output = new ConverseOutput + { + Message = new Message + { + Role = ConversationRole.Assistant, + Content = new List + { + new ContentBlock + { + ToolUse = new ToolUseBlock + { + ToolUseId = "test-id", + Name = "generate_response", + Input = new Document(new Dictionary + { + ["name"] = new Document("John Doe"), + ["age"] = new Document(30) + }) + } + } + } + } + }, + StopReason = new StopReason("tool_use") + }); + + var client = mockRuntime.Object.AsIChatClient("claude-3"); var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; var schemaJson = """ @@ -172,17 +138,11 @@ public async Task ResponseFormat_Json_WithSchema_CreatesSyntheticToolWithCorrect }; // Act - try - { - await client.GetResponseAsync(messages, options); - } - catch - { - // We're testing request creation - } + await client.GetResponseAsync(messages, options); // Assert - var tool = mock.CapturedRequest.ToolConfig.Tools[0]; + Assert.NotNull(capturedRequest); + var tool = capturedRequest.ToolConfig.Tools[0]; Assert.Equal("generate_response", tool.ToolSpec.Name); Assert.Equal("A person object", tool.ToolSpec.Description); @@ -204,6 +164,9 @@ public async Task ResponseFormat_Json_WithSchema_CreatesSyntheticToolWithCorrect var required = schemaDict["required"].AsList(); Assert.Single(required); Assert.Equal("name", required[0].AsString()); + + // Verify the mock was called + mockRuntime.Verify(x => x.ConverseAsync(It.IsAny(), It.IsAny()), Times.Once); } [Fact] @@ -211,41 +174,44 @@ public async Task ResponseFormat_Json_WithSchema_CreatesSyntheticToolWithCorrect public async Task ResponseFormat_Json_ModelReturnsToolUse_ExtractsJsonCorrectly() { // Arrange - var mock = new MockBedrockRuntime(); - var client = mock.AsIChatClient("claude-3"); - var messages = new[] { new ChatMessage(ChatRole.User, "Get weather") }; - var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; + var mockRuntime = new Mock(); // Setup mock to return tool use with structured data - mock.ResponseFactory = req => new ConverseResponse - { - Output = new ConverseOutput + mockRuntime + .Setup(x => x.ConverseAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new ConverseResponse { - Message = new Message + Output = new ConverseOutput { - Role = ConversationRole.Assistant, - Content = new List + Message = new Message { - new ContentBlock + Role = ConversationRole.Assistant, + Content = new List { - ToolUse = new ToolUseBlock + new ContentBlock { - ToolUseId = "test-id", - Name = "generate_response", - Input = new Document(new Dictionary + ToolUse = new ToolUseBlock { - ["city"] = new Document("Seattle"), - ["temperature"] = new Document(72), - ["conditions"] = new Document("sunny") - }) + ToolUseId = "test-id", + Name = "generate_response", + Input = new Document(new Dictionary + { + ["city"] = new Document("Seattle"), + ["temperature"] = new Document(72), + ["conditions"] = new Document("sunny") + }) + } } } } - } - }, - StopReason = new StopReason("tool_use"), - Usage = new TokenUsage { InputTokens = 10, OutputTokens = 20, TotalTokens = 30 } - }; + }, + StopReason = new StopReason("tool_use"), + Usage = new TokenUsage { InputTokens = 10, OutputTokens = 20, TotalTokens = 30 } + }); + + var client = mockRuntime.Object.AsIChatClient("claude-3"); + var messages = new[] { new ChatMessage(ChatRole.User, "Get weather") }; + var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; // Act var response = await client.GetResponseAsync(messages, options); @@ -261,5 +227,949 @@ public async Task ResponseFormat_Json_ModelReturnsToolUse_ExtractsJsonCorrectly( Assert.Equal("sunny", json.RootElement.GetProperty("conditions").GetString()); } + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Json_WithTools_ThrowsArgumentException() + { + // Arrange + var mockRuntime = new Mock(); + var client = mockRuntime.Object.AsIChatClient("claude-3"); + var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; + + // Create test tool + var tool = new TestAIFunction("test", "Test tool", JsonDocument.Parse("{}").RootElement); + + var options = new ChatOptions + { + ResponseFormat = ChatResponseFormat.Json, + Tools = new[] { tool } + }; + + // Act & Assert + await Assert.ThrowsAsync(async () => + await client.GetResponseAsync(messages, options)); + } + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Json_UnsupportedModel_ThrowsNotSupportedException() + { + // Arrange + var mockRuntime = new Mock(); + + // Setup mock to throw BedrockRuntimeException with toolChoice error + mockRuntime + .Setup(x => x.ConverseAsync(It.IsAny(), It.IsAny())) + .ThrowsAsync(new AmazonBedrockRuntimeException("ValidationException: toolChoice is not supported by this model") + { + ErrorCode = "ValidationException" + }); + + var client = mockRuntime.Object.AsIChatClient("titan"); + var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; + var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; + + // Act & Assert + var ex = await Assert.ThrowsAsync(async () => + await client.GetResponseAsync(messages, options)); + + Assert.Contains("does not support ResponseFormat", ex.Message); + Assert.Contains("ToolChoice", ex.Message); + } + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Json_ForStreaming_ThrowsNotSupportedException() + { + // Arrange + var mockRuntime = new Mock(); + var client = mockRuntime.Object.AsIChatClient("claude-3"); + var messages = new[] { new ChatMessage(ChatRole.User, "Test") }; + var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; + + // Act & Assert + await Assert.ThrowsAsync(async () => + { + await foreach (var update in client.GetStreamingResponseAsync(messages, options)) + { + // Should not reach here + } + }); + } + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Json_ModelReturnsText_ThrowsInvalidOperationException() + { + // Arrange - Model returns text instead of tool_use + var mockRuntime = new Mock(); + + mockRuntime + .Setup(x => x.ConverseAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new ConverseResponse + { + Output = new ConverseOutput + { + Message = new Message + { + Role = ConversationRole.Assistant, + Content = new List + { + new ContentBlock { Text = "Here is some text" } + } + } + }, + StopReason = new StopReason("end_turn") + }); + + var client = mockRuntime.Object.AsIChatClient("claude-3"); + var messages = new[] { new ChatMessage(ChatRole.User, "Generate data") }; + var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; + + // Act & Assert + var ex = await Assert.ThrowsAsync(async () => + await client.GetResponseAsync(messages, options)); + + Assert.Contains("did not return structured output", ex.Message); + Assert.Contains("end_turn", ex.Message); + } + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Json_WrongToolName_ThrowsInvalidOperationException() + { + // Arrange - Model uses wrong tool name + var mockRuntime = new Mock(); + + mockRuntime + .Setup(x => x.ConverseAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new ConverseResponse + { + Output = new ConverseOutput + { + Message = new Message + { + Role = ConversationRole.Assistant, + Content = new List + { + new ContentBlock + { + ToolUse = new ToolUseBlock + { + ToolUseId = "wrong-id", + Name = "wrong_tool_name", + Input = new Document(new Dictionary + { + ["data"] = new Document("value") + }) + } + } + } + } + }, + StopReason = new StopReason("tool_use") + }); + + var client = mockRuntime.Object.AsIChatClient("claude-3"); + var messages = new[] { new ChatMessage(ChatRole.User, "Generate data") }; + var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; + + // Act & Assert + var ex = await Assert.ThrowsAsync(async () => + await client.GetResponseAsync(messages, options)); + + Assert.Contains("did not return structured output", ex.Message); + } + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Json_EmptyToolInput_ReturnsEmptyJson() + { + // Arrange - Tool with empty input + var mockRuntime = new Mock(); + + mockRuntime + .Setup(x => x.ConverseAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new ConverseResponse + { + Output = new ConverseOutput + { + Message = new Message + { + Role = ConversationRole.Assistant, + Content = new List + { + new ContentBlock + { + ToolUse = new ToolUseBlock + { + ToolUseId = "empty-id", + Name = "generate_response", + Input = new Document(new Dictionary()) + } + } + } + } + }, + StopReason = new StopReason("tool_use") + }); + + var client = mockRuntime.Object.AsIChatClient("claude-3"); + var messages = new[] { new ChatMessage(ChatRole.User, "Generate data") }; + var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Json }; + + // Act + var response = await client.GetResponseAsync(messages, options); + + // Assert - Empty object is valid JSON + Assert.NotNull(response.Text); + var json = JsonDocument.Parse(response.Text); + Assert.Equal(JsonValueKind.Object, json.RootElement.ValueKind); + } + #endregion } + +/// +/// Tests using HTTP-layer mocking to test actual Converse API response scenarios. +/// This allows testing beyond the happy path with realistic service responses. +/// Based on Peter's (peterrsongg) suggestion to test different response structures. +/// +public class BedrockChatClientHttpMockedTests : IClassFixture +{ + private readonly HttpMockFixture _fixture; + + public BedrockChatClientHttpMockedTests(HttpMockFixture fixture) + { + _fixture = fixture; + } + + /// + /// Helper method to inject stubbed web response data into a request's state + /// + private static void InjectMockedResponse(ConverseRequest request, StubWebResponseData webResponseData) + { + var interfaceType = typeof(IAmazonWebServiceRequest); + var requestStatePropertyInfo = interfaceType.GetProperty("RequestState"); + var requestState = (Dictionary)requestStatePropertyInfo.GetValue(request); + requestState["response"] = webResponseData; + } + + #region HTTP Mocking Infrastructure (Based on Peter's Working Code) + + /// + /// Pipeline customizer that replaces the HTTP handler with a mock implementation + /// + private class MockPipelineCustomizer : IRuntimePipelineCustomizer + { + public string UniqueName => "BedrockMEAIMockPipeline"; + + public void Customize(Type type, RuntimePipeline pipeline) + { +#if BCL + // On .NET Framework, use Stream + pipeline.ReplaceHandler>( + new HttpHandler(new MockHttpRequestFactory(), new object())); +#else + // On .NET Core/.NET 5+, use HttpContent + pipeline.ReplaceHandler>( + new HttpHandler(new MockHttpRequestFactory(), new object())); +#endif + } + } + + /// + /// Factory for creating mock HTTP requests + /// +#if BCL + private class MockHttpRequestFactory : IHttpRequestFactory + { + public IHttpRequest CreateHttpRequest(Uri requestUri) + { + return new MockHttpRequest(requestUri); + } +#else + private class MockHttpRequestFactory : IHttpRequestFactory + { + public IHttpRequest CreateHttpRequest(Uri requestUri) + { + return new MockHttpRequest(requestUri); + } +#endif + + public void Dispose() + { + // No resources to dispose + } + } + + /// + /// Mock HTTP request that retrieves stubbed response data from request state + /// +#if BCL + private class MockHttpRequest : IHttpRequest +#else + private class MockHttpRequest : IHttpRequest +#endif + { + private IWebResponseData _webResponseData; + + public MockHttpRequest(Uri requestUri) + { + RequestUri = requestUri; + } + + public string Method { get; set; } + public Uri RequestUri { get; set; } + public Version HttpProtocolVersion { get; set; } + + public void ConfigureRequest(IRequestContext requestContext) + { + // Retrieve the stubbed response from request state + // This is the critical line that Peter identified (line 60 in his comment) + var request = requestContext.OriginalRequest as IAmazonWebServiceRequest; + if (request != null && request.RequestState.ContainsKey("response")) + { + _webResponseData = request.RequestState["response"] as IWebResponseData; + } + } + + public void SetRequestHeaders(IDictionary headers) + { + // Not needed for mock + } + +#if BCL + public Stream GetRequestContent() + { + return new MemoryStream(); + } +#else + public HttpContent GetRequestContent() + { + return null; + } +#endif + + public IWebResponseData GetResponse() + { + return GetResponseAsync(CancellationToken.None).Result; + } + + public Task GetResponseAsync(CancellationToken cancellationToken) + { + return Task.FromResult(_webResponseData); + } + +#if BCL + public void WriteToRequestBody(Stream requestContent, Stream contentStream, + IDictionary contentHeaders, IRequestContext requestContext) + { + // Not needed for mock + } + + public void WriteToRequestBody(Stream requestContent, byte[] content, + IDictionary contentHeaders) + { + // Not needed for mock + } + + public Task WriteToRequestBodyAsync(Stream requestContent, Stream contentStream, + IDictionary contentHeaders, IRequestContext requestContext) + { + return Task.CompletedTask; + } + + public Task WriteToRequestBodyAsync(Stream requestContent, byte[] content, + IDictionary contentHeaders, CancellationToken cancellationToken = default) + { + return Task.CompletedTask; + } +#else + public void WriteToRequestBody(HttpContent requestContent, Stream contentStream, + IDictionary contentHeaders, IRequestContext requestContext) + { + // Not needed for mock + } + + public void WriteToRequestBody(HttpContent requestContent, byte[] content, + IDictionary contentHeaders) + { + // Not needed for mock + } + + public Task WriteToRequestBodyAsync(HttpContent requestContent, Stream contentStream, + IDictionary contentHeaders, IRequestContext requestContext) + { + return Task.CompletedTask; + } + + public Task WriteToRequestBodyAsync(HttpContent requestContent, byte[] content, + IDictionary contentHeaders, CancellationToken cancellationToken = default) + { + return Task.CompletedTask; + } +#endif + + public IHttpRequestStreamHandle SetupHttpRequestStreamPublisher( + IDictionary contentHeaders, IHttpRequestStreamPublisher publisher) + { + throw new NotImplementedException(); + } + + public void Abort() + { + // Not needed for mock + } + +#if BCL + public Task GetRequestContentAsync() + { + return Task.FromResult(new MemoryStream()); + } + + public Task GetRequestContentAsync(CancellationToken cancellationToken) + { + return Task.FromResult(new MemoryStream()); + } +#else + public Task GetRequestContentAsync() + { + return Task.FromResult(null); + } + + public Task GetRequestContentAsync(CancellationToken cancellationToken) + { + return Task.FromResult(null); + } +#endif + + public Stream SetupProgressListeners(Stream originalStream, long progressUpdateInterval, + object sender, EventHandler callback) + { + return originalStream; + } + + public void Dispose() + { + // Nothing to dispose + } + } + + /// + /// Stubbed web response data for testing different response scenarios + /// + private class StubWebResponseData : IWebResponseData + { + private readonly IHttpResponseBody _httpResponseBody; + + public StubWebResponseData(string jsonResponse, Dictionary headers = null, + HttpStatusCode statusCode = HttpStatusCode.OK) + { + StatusCode = statusCode; + IsSuccessStatusCode = (int)statusCode >= 200 && (int)statusCode < 300; + JsonResponse = jsonResponse; + Headers = headers ?? new Dictionary(StringComparer.OrdinalIgnoreCase); + ContentType = "application/json"; + ContentLength = jsonResponse?.Length ?? 0; + + _httpResponseBody = new HttpResponseBody(jsonResponse); + } + + public Dictionary Headers { get; set; } + public string JsonResponse { get; } + public long ContentLength { get; set; } + public string ContentType { get; set; } + public HttpStatusCode StatusCode { get; set; } + public bool IsSuccessStatusCode { get; set; } + + public IHttpResponseBody ResponseBody => _httpResponseBody; + + public string[] GetHeaderNames() + { + return Headers.Keys.ToArray(); + } + + public bool IsHeaderPresent(string headerName) + { + return Headers.ContainsKey(headerName); + } + + public string GetHeaderValue(string headerName) + { + return Headers.ContainsKey(headerName) ? Headers[headerName] : null; + } + } + + /// + /// HTTP response body implementation for stubbed responses + /// + private class HttpResponseBody : IHttpResponseBody + { + private readonly string _jsonResponse; + private Stream _stream; + + public HttpResponseBody(string jsonResponse) + { + _jsonResponse = jsonResponse ?? string.Empty; + } + + public void Dispose() + { + _stream?.Dispose(); + } + + public Stream OpenResponse() + { + _stream = new MemoryStream(Encoding.UTF8.GetBytes(_jsonResponse)); + return _stream; + } + + public Task OpenResponseAsync() + { + return Task.FromResult(OpenResponse()); + } + } + + #endregion + + #region ResponseFormat with HTTP Mocking Tests + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Json_WithActualConverseResponse_ParsesCorrectly() + { + // Arrange - This is a real Converse API response with tool_use + var converseResponse = """ + { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "toolUse": { + "toolUseId": "tooluse_12345", + "name": "generate_response", + "input": { + "name": "Alice Johnson", + "age": 28, + "city": "Seattle" + } + } + } + ] + } + }, + "stopReason": "tool_use", + "usage": { + "inputTokens": 125, + "outputTokens": 45, + "totalTokens": 170 + } + } + """; + + var chatClient = _fixture.BedrockRuntimeClient.AsIChatClient("anthropic.claude-3-sonnet-20240229-v1:0"); + var messages = new[] { new ChatMessage(ChatRole.User, "Generate a person") }; + + var schemaJson = """ + { + "type": "object", + "properties": { + "name": { "type": "string" }, + "age": { "type": "number" }, + "city": { "type": "string" } + }, + "required": ["name", "age"] + } + """; + var schemaElement = JsonDocument.Parse(schemaJson).RootElement; + + var request = new ConverseRequest(); + var options = new ChatOptions + { + ResponseFormat = ChatResponseFormat.ForJsonSchema(schemaElement, + schemaName: "PersonSchema", + schemaDescription: "A person with demographic information"), + RawRepresentationFactory = _ => request + }; + + // Inject the stubbed response + var webResponseData = new StubWebResponseData(converseResponse); + InjectMockedResponse(request, webResponseData); + + // Act + var response = await chatClient.GetResponseAsync(messages, options); + + // Assert + Assert.NotNull(response); + Assert.NotNull(response.Text); + + // Verify the JSON structure + var json = JsonDocument.Parse(response.Text); + Assert.Equal("Alice Johnson", json.RootElement.GetProperty("name").GetString()); + Assert.Equal(28, json.RootElement.GetProperty("age").GetInt32()); + Assert.Equal("Seattle", json.RootElement.GetProperty("city").GetString()); + + // Verify usage metadata + var usage = response.Usage; + Assert.NotNull(usage); + Assert.Equal(125, usage.InputTokenCount); + Assert.Equal(45, usage.OutputTokenCount); + Assert.Equal(170, usage.TotalTokenCount); + } + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Json_WithNestedObjects_ParsesCorrectly() + { + // Arrange - Test with nested JSON structure + var converseResponse = """ + { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "toolUse": { + "toolUseId": "tooluse_nested", + "name": "generate_response", + "input": { + "user": { + "name": "Bob Smith", + "contact": { + "email": "bob@example.com", + "phone": "555-0123" + } + }, + "metadata": { + "timestamp": "2024-01-15T10:30:00Z", + "version": 1 + } + } + } + } + ] + } + }, + "stopReason": "tool_use", + "usage": { + "inputTokens": 200, + "outputTokens": 80, + "totalTokens": 280 + } + } + """; + + var chatClient = _fixture.BedrockRuntimeClient.AsIChatClient("anthropic.claude-3-sonnet-20240229-v1:0"); + var messages = new[] { new ChatMessage(ChatRole.User, "Generate user data") }; + + var request = new ConverseRequest(); + var options = new ChatOptions + { + ResponseFormat = ChatResponseFormat.Json, + RawRepresentationFactory = _ => request + }; + + var webResponseData = new StubWebResponseData(converseResponse); + InjectMockedResponse(request, webResponseData); + + // Act + var response = await chatClient.GetResponseAsync(messages, options); + + // Assert + Assert.NotNull(response.Text); + var json = JsonDocument.Parse(response.Text); + + var user = json.RootElement.GetProperty("user"); + Assert.Equal("Bob Smith", user.GetProperty("name").GetString()); + + var contact = user.GetProperty("contact"); + Assert.Equal("bob@example.com", contact.GetProperty("email").GetString()); + Assert.Equal("555-0123", contact.GetProperty("phone").GetString()); + + var metadata = json.RootElement.GetProperty("metadata"); + Assert.Equal("2024-01-15T10:30:00Z", metadata.GetProperty("timestamp").GetString()); + Assert.Equal(1, metadata.GetProperty("version").GetInt32()); + } + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Json_WithArrayData_ParsesCorrectly() + { + // Arrange - Test with arrays in JSON response + var converseResponse = """ + { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "toolUse": { + "toolUseId": "tooluse_array", + "name": "generate_response", + "input": { + "items": ["apple", "banana", "orange"], + "prices": [1.99, 0.99, 2.49], + "inventory": { + "warehouse": "A", + "quantities": [100, 250, 75] + } + } + } + } + ] + } + }, + "stopReason": "tool_use", + "usage": { + "inputTokens": 50, + "outputTokens": 30, + "totalTokens": 80 + } + } + """; + + var chatClient = _fixture.BedrockRuntimeClient.AsIChatClient("anthropic.claude-3-sonnet-20240229-v1:0"); + var messages = new[] { new ChatMessage(ChatRole.User, "List items") }; + + var request = new ConverseRequest(); + var options = new ChatOptions + { + ResponseFormat = ChatResponseFormat.Json, + RawRepresentationFactory = _ => request + }; + + var webResponseData = new StubWebResponseData(converseResponse); + InjectMockedResponse(request, webResponseData); + + // Act + var response = await chatClient.GetResponseAsync(messages, options); + + // Assert + Assert.NotNull(response.Text); + var json = JsonDocument.Parse(response.Text); + + var items = json.RootElement.GetProperty("items"); + Assert.Equal(JsonValueKind.Array, items.ValueKind); + Assert.Equal(3, items.GetArrayLength()); + Assert.Equal("apple", items[0].GetString()); + Assert.Equal("banana", items[1].GetString()); + Assert.Equal("orange", items[2].GetString()); + + var prices = json.RootElement.GetProperty("prices"); + Assert.Equal(3, prices.GetArrayLength()); + Assert.Equal(1.99, prices[0].GetDouble(), precision: 2); + + var inventory = json.RootElement.GetProperty("inventory"); + var quantities = inventory.GetProperty("quantities"); + Assert.Equal(3, quantities.GetArrayLength()); + Assert.Equal(100, quantities[0].GetInt32()); + } + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Json_WithMinimalSchema_ParsesCorrectly() + { + // Arrange - Test simple JSON response + var converseResponse = """ + { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "toolUse": { + "toolUseId": "tooluse_simple", + "name": "generate_response", + "input": { + "message": "Hello, World!", + "status": "success" + } + } + } + ] + } + }, + "stopReason": "tool_use", + "usage": { + "inputTokens": 10, + "outputTokens": 5, + "totalTokens": 15 + } + } + """; + + var chatClient = _fixture.BedrockRuntimeClient.AsIChatClient("anthropic.claude-3-haiku-20240307-v1:0"); + var messages = new[] { new ChatMessage(ChatRole.User, "Say hello") }; + + var request = new ConverseRequest(); + var options = new ChatOptions + { + ResponseFormat = ChatResponseFormat.Json, + RawRepresentationFactory = _ => request + }; + + var webResponseData = new StubWebResponseData(converseResponse); + InjectMockedResponse(request, webResponseData); + + // Act + var response = await chatClient.GetResponseAsync(messages, options); + + // Assert + Assert.NotNull(response.Text); + var json = JsonDocument.Parse(response.Text); + Assert.Equal("Hello, World!", json.RootElement.GetProperty("message").GetString()); + Assert.Equal("success", json.RootElement.GetProperty("status").GetString()); + } + + [Fact] + [Trait("UnitTest", "BedrockRuntime")] + public async Task ResponseFormat_Json_WithComplexSchema_ValidatesStructure() + { + // Arrange - Test with detailed schema validation + var converseResponse = """ + { + "output": { + "message": { + "role": "assistant", + "content": [ + { + "toolUse": { + "toolUseId": "tooluse_complex", + "name": "generate_response", + "input": { + "id": "usr_123", + "username": "testuser", + "email": "test@example.com", + "profile": { + "firstName": "Test", + "lastName": "User", + "age": 25, + "preferences": { + "theme": "dark", + "notifications": true + } + }, + "roles": ["admin", "user"], + "active": true + } + } + } + ] + } + }, + "stopReason": "tool_use", + "usage": { + "inputTokens": 300, + "outputTokens": 150, + "totalTokens": 450 + } + } + """; + + var chatClient = _fixture.BedrockRuntimeClient.AsIChatClient("anthropic.claude-3-sonnet-20240229-v1:0"); + var messages = new[] { new ChatMessage(ChatRole.User, "Generate user profile") }; + + var schemaJson = """ + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "username": { "type": "string" }, + "email": { "type": "string", "format": "email" }, + "profile": { + "type": "object", + "properties": { + "firstName": { "type": "string" }, + "lastName": { "type": "string" }, + "age": { "type": "number" }, + "preferences": { "type": "object" } + }, + "required": ["firstName", "lastName"] + }, + "roles": { + "type": "array", + "items": { "type": "string" } + }, + "active": { "type": "boolean" } + }, + "required": ["id", "username", "email"] + } + """; + var schemaElement = JsonDocument.Parse(schemaJson).RootElement; + + var request = new ConverseRequest(); + var options = new ChatOptions + { + ResponseFormat = ChatResponseFormat.ForJsonSchema(schemaElement, + schemaName: "UserProfile", + schemaDescription: "Complete user profile with preferences"), + RawRepresentationFactory = _ => request + }; + + var webResponseData = new StubWebResponseData(converseResponse); + InjectMockedResponse(request, webResponseData); + + // Act + var response = await chatClient.GetResponseAsync(messages, options); + + // Assert + Assert.NotNull(response.Text); + var json = JsonDocument.Parse(response.Text); + + // Verify required fields + Assert.Equal("usr_123", json.RootElement.GetProperty("id").GetString()); + Assert.Equal("testuser", json.RootElement.GetProperty("username").GetString()); + Assert.Equal("test@example.com", json.RootElement.GetProperty("email").GetString()); + + // Verify nested profile + var profile = json.RootElement.GetProperty("profile"); + Assert.Equal("Test", profile.GetProperty("firstName").GetString()); + Assert.Equal("User", profile.GetProperty("lastName").GetString()); + Assert.Equal(25, profile.GetProperty("age").GetInt32()); + + // Verify nested preferences + var preferences = profile.GetProperty("preferences"); + Assert.Equal("dark", preferences.GetProperty("theme").GetString()); + Assert.True(preferences.GetProperty("notifications").GetBoolean()); + + // Verify array + var roles = json.RootElement.GetProperty("roles"); + Assert.Equal(2, roles.GetArrayLength()); + Assert.Equal("admin", roles[0].GetString()); + Assert.Equal("user", roles[1].GetString()); + + // Verify boolean + Assert.True(json.RootElement.GetProperty("active").GetBoolean()); + } + + #endregion + + /// + /// Test fixture that registers the HTTP mocking pipeline customizer + /// + public class HttpMockFixture : IDisposable + { + private readonly MockPipelineCustomizer _customizer; + + public HttpMockFixture() + { + // Register the mock pipeline customizer globally + _customizer = new MockPipelineCustomizer(); + Runtime.Internal.RuntimePipelineCustomizerRegistry.Instance.Register(_customizer); + + // Create the Bedrock Runtime client - it will use the mocked pipeline + BedrockRuntimeClient = new AmazonBedrockRuntimeClient(); + } + + public IAmazonBedrockRuntime BedrockRuntimeClient { get; private set; } + + public void Dispose() + { + // Clean up + Runtime.Internal.RuntimePipelineCustomizerRegistry.Instance.Deregister(_customizer); + BedrockRuntimeClient?.Dispose(); + } + } +} \ No newline at end of file From 549170ef448d193aa92abc13dcf0dc59d95504ec Mon Sep 17 00:00:00 2001 From: adaines Date: Wed, 19 Nov 2025 13:06:59 -0500 Subject: [PATCH 5/7] add devconfig --- .../12b83a1f-1d6b-4e96-bd62-f0e0b7e4df6d.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 generator/.DevConfigs/12b83a1f-1d6b-4e96-bd62-f0e0b7e4df6d.json diff --git a/generator/.DevConfigs/12b83a1f-1d6b-4e96-bd62-f0e0b7e4df6d.json b/generator/.DevConfigs/12b83a1f-1d6b-4e96-bd62-f0e0b7e4df6d.json new file mode 100644 index 000000000000..297c2483daf0 --- /dev/null +++ b/generator/.DevConfigs/12b83a1f-1d6b-4e96-bd62-f0e0b7e4df6d.json @@ -0,0 +1,11 @@ +{ + "extensions": [ + { + "extensionName": "Extensions.Bedrock.MEAI", + "type": "minor", + "changeLogMessages": [ + "Add support for ChatOptions.ResponseFormat to enable structured JSON responses using JSON Schema." + ] + } + ] +} From b7bd419ec9c1528ae7b37e4c12e4544aba34640b Mon Sep 17 00:00:00 2001 From: adaines Date: Wed, 19 Nov 2025 17:15:17 -0500 Subject: [PATCH 6/7] fix: add dependency for moq --- .../test/BedrockMEAITests/BedrockMEAITests.NetFramework.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/extensions/test/BedrockMEAITests/BedrockMEAITests.NetFramework.csproj b/extensions/test/BedrockMEAITests/BedrockMEAITests.NetFramework.csproj index dd9de35ce4a5..915b0769fe83 100644 --- a/extensions/test/BedrockMEAITests/BedrockMEAITests.NetFramework.csproj +++ b/extensions/test/BedrockMEAITests/BedrockMEAITests.NetFramework.csproj @@ -19,6 +19,7 @@ + From 302dc792c74e62d509e7afc46d75fa64e5798fa6 Mon Sep 17 00:00:00 2001 From: adaines Date: Thu, 20 Nov 2025 00:26:59 -0500 Subject: [PATCH 7/7] define bcl for compatibility with .net framework --- .../test/BedrockMEAITests/BedrockMEAITests.NetFramework.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/extensions/test/BedrockMEAITests/BedrockMEAITests.NetFramework.csproj b/extensions/test/BedrockMEAITests/BedrockMEAITests.NetFramework.csproj index 915b0769fe83..1289d63cea0d 100644 --- a/extensions/test/BedrockMEAITests/BedrockMEAITests.NetFramework.csproj +++ b/extensions/test/BedrockMEAITests/BedrockMEAITests.NetFramework.csproj @@ -1,6 +1,7 @@  net472 + $(DefineConstants);BCL BedrockMEAITests BedrockMEAITests