diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index 15b86590df..6138cd60ef 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -302,6 +302,8 @@ + + diff --git a/dotnet/samples/AzureFunctions/01_SingleAgent/Program.cs b/dotnet/samples/AzureFunctions/01_SingleAgent/Program.cs index ebb5082893..60b3103adc 100644 --- a/dotnet/samples/AzureFunctions/01_SingleAgent/Program.cs +++ b/dotnet/samples/AzureFunctions/01_SingleAgent/Program.cs @@ -16,7 +16,7 @@ ?? throw new InvalidOperationException("AZURE_OPENAI_DEPLOYMENT is not set."); // Use Azure Key Credential if provided, otherwise use Azure CLI Credential. -string? azureOpenAiKey = Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY"); +string? azureOpenAiKey = System.Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY"); AzureOpenAIClient client = !string.IsNullOrEmpty(azureOpenAiKey) ? new AzureOpenAIClient(new Uri(endpoint), new AzureKeyCredential(azureOpenAiKey)) : new AzureOpenAIClient(new Uri(endpoint), new AzureCliCredential()); diff --git a/dotnet/samples/AzureFunctions/02_AgentOrchestration_Chaining/Program.cs b/dotnet/samples/AzureFunctions/02_AgentOrchestration_Chaining/Program.cs index 14abd390c9..41f643a763 100644 --- a/dotnet/samples/AzureFunctions/02_AgentOrchestration_Chaining/Program.cs +++ b/dotnet/samples/AzureFunctions/02_AgentOrchestration_Chaining/Program.cs @@ -16,7 +16,7 @@ ?? throw new InvalidOperationException("AZURE_OPENAI_DEPLOYMENT is not set."); // Use Azure Key Credential if provided, otherwise use Azure CLI Credential. -string? azureOpenAiKey = Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY"); +string? azureOpenAiKey = System.Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY"); AzureOpenAIClient client = !string.IsNullOrEmpty(azureOpenAiKey) ? new AzureOpenAIClient(new Uri(endpoint), new AzureKeyCredential(azureOpenAiKey)) : new AzureOpenAIClient(new Uri(endpoint), new AzureCliCredential()); diff --git a/dotnet/samples/AzureFunctions/03_AgentOrchestration_Concurrency/Program.cs b/dotnet/samples/AzureFunctions/03_AgentOrchestration_Concurrency/Program.cs index e1bd92d3e4..d4d5750df7 100644 --- a/dotnet/samples/AzureFunctions/03_AgentOrchestration_Concurrency/Program.cs +++ b/dotnet/samples/AzureFunctions/03_AgentOrchestration_Concurrency/Program.cs @@ -16,7 +16,7 @@ ?? throw new InvalidOperationException("AZURE_OPENAI_DEPLOYMENT is not set."); // Use Azure Key Credential if provided, otherwise use Azure CLI Credential. -string? azureOpenAiKey = Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY"); +string? azureOpenAiKey = System.Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY"); AzureOpenAIClient client = !string.IsNullOrEmpty(azureOpenAiKey) ? new AzureOpenAIClient(new Uri(endpoint), new AzureKeyCredential(azureOpenAiKey)) : new AzureOpenAIClient(new Uri(endpoint), new AzureCliCredential()); diff --git a/dotnet/samples/AzureFunctions/04_AgentOrchestration_Conditionals/Program.cs b/dotnet/samples/AzureFunctions/04_AgentOrchestration_Conditionals/Program.cs index ac14105737..e63d1a9667 100644 --- a/dotnet/samples/AzureFunctions/04_AgentOrchestration_Conditionals/Program.cs +++ b/dotnet/samples/AzureFunctions/04_AgentOrchestration_Conditionals/Program.cs @@ -16,7 +16,7 @@ ?? throw new InvalidOperationException("AZURE_OPENAI_DEPLOYMENT is not set."); // Use Azure Key Credential if provided, otherwise use Azure CLI Credential. -string? azureOpenAiKey = Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY"); +string? azureOpenAiKey = System.Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY"); AzureOpenAIClient client = !string.IsNullOrEmpty(azureOpenAiKey) ? new AzureOpenAIClient(new Uri(endpoint), new AzureKeyCredential(azureOpenAiKey)) : new AzureOpenAIClient(new Uri(endpoint), new AzureCliCredential()); diff --git a/dotnet/samples/AzureFunctions/05_AgentOrchestration_HITL/Program.cs b/dotnet/samples/AzureFunctions/05_AgentOrchestration_HITL/Program.cs index 8287ad5f30..457fc4936e 100644 --- a/dotnet/samples/AzureFunctions/05_AgentOrchestration_HITL/Program.cs +++ b/dotnet/samples/AzureFunctions/05_AgentOrchestration_HITL/Program.cs @@ -16,7 +16,7 @@ ?? throw new InvalidOperationException("AZURE_OPENAI_DEPLOYMENT is not set."); // Use Azure Key Credential if provided, otherwise use Azure CLI Credential. -string? azureOpenAiKey = Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY"); +string? azureOpenAiKey = System.Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY"); AzureOpenAIClient client = !string.IsNullOrEmpty(azureOpenAiKey) ? new AzureOpenAIClient(new Uri(endpoint), new AzureKeyCredential(azureOpenAiKey)) : new AzureOpenAIClient(new Uri(endpoint), new AzureCliCredential()); diff --git a/dotnet/samples/AzureFunctions/06_LongRunningTools/Program.cs b/dotnet/samples/AzureFunctions/06_LongRunningTools/Program.cs index fe7446c842..657a80d21f 100644 --- a/dotnet/samples/AzureFunctions/06_LongRunningTools/Program.cs +++ b/dotnet/samples/AzureFunctions/06_LongRunningTools/Program.cs @@ -20,7 +20,7 @@ ?? throw new InvalidOperationException("AZURE_OPENAI_DEPLOYMENT is not set."); // Use Azure Key Credential if provided, otherwise use Azure CLI Credential. -string? azureOpenAiKey = Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY"); +string? azureOpenAiKey = System.Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY"); AzureOpenAIClient client = !string.IsNullOrEmpty(azureOpenAiKey) ? new AzureOpenAIClient(new Uri(endpoint), new AzureKeyCredential(azureOpenAiKey)) : new AzureOpenAIClient(new Uri(endpoint), new AzureCliCredential()); diff --git a/dotnet/samples/AzureFunctions/07_AgentAsMcpTool/Program.cs b/dotnet/samples/AzureFunctions/07_AgentAsMcpTool/Program.cs index 40dffe21a0..1c55f41f16 100644 --- a/dotnet/samples/AzureFunctions/07_AgentAsMcpTool/Program.cs +++ b/dotnet/samples/AzureFunctions/07_AgentAsMcpTool/Program.cs @@ -22,7 +22,7 @@ ?? throw new InvalidOperationException("AZURE_OPENAI_DEPLOYMENT is not set."); // Use Azure Key Credential if provided, otherwise use Azure CLI Credential. -string? azureOpenAiKey = Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY"); +string? azureOpenAiKey = System.Environment.GetEnvironmentVariable("AZURE_OPENAI_KEY"); AzureOpenAIClient client = !string.IsNullOrEmpty(azureOpenAiKey) ? new AzureOpenAIClient(new Uri(endpoint), new AzureKeyCredential(azureOpenAiKey)) : new AzureOpenAIClient(new Uri(endpoint), new AzureCliCredential()); diff --git a/dotnet/src/Microsoft.Agents.AI.DurableTask/README.md b/dotnet/src/Microsoft.Agents.AI.DurableTask/README.md index ede4f19051..87e4091126 100644 --- a/dotnet/src/Microsoft.Agents.AI.DurableTask/README.md +++ b/dotnet/src/Microsoft.Agents.AI.DurableTask/README.md @@ -1,6 +1,6 @@ # Microsoft.Agents.AI.DurableTask -The Microsoft Agent Framework provides a programming model for building agents and agent workflows in .NET. This package, the *Durable extensions for the Agent Framework*, extends the Agent Framework programming model with the following capabilities: +The Microsoft Agent Framework provides a programming model for building agents and agent workflows in .NET. This package, the *Durable Task extension for the Agent Framework*, extends the Agent Framework programming model with the following capabilities: - Stateful, durable execution of agents in distributed environments - Automatic conversation history management diff --git a/dotnet/src/Microsoft.Agents.AI.Hosting.AzureFunctions/README.md b/dotnet/src/Microsoft.Agents.AI.Hosting.AzureFunctions/README.md index 7536790a97..14bd297387 100644 --- a/dotnet/src/Microsoft.Agents.AI.Hosting.AzureFunctions/README.md +++ b/dotnet/src/Microsoft.Agents.AI.Hosting.AzureFunctions/README.md @@ -25,7 +25,7 @@ Or directly in your project file: ## Usage Examples -For a comprehensive tour of all the functionality, concepts, and APIs, check out the [Azure Functions samples](https://github.com/microsoft/agent-framework/tree/main/dotnet/samples/AzureFunctions) in the [Durable extensions for Agent Framework repository](https://github.com/microsoft/agent-framework). +For a comprehensive tour of all the functionality, concepts, and APIs, check out the [Azure Functions samples](https://github.com/microsoft/agent-framework/tree/main/dotnet/samples/AzureFunctions) in the [Durable Task extension for Agent Framework repository](https://github.com/microsoft/agent-framework). ### Hosting single agents diff --git a/dotnet/tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/AgentEntityTests.cs b/dotnet/tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/AgentEntityTests.cs new file mode 100644 index 0000000000..73c230410c --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/AgentEntityTests.cs @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics; +using System.Reflection; +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.Entities; +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Configuration; +using OpenAI; +using Xunit.Abstractions; + +namespace Microsoft.Agents.AI.DurableTask.IntegrationTests; + +/// +/// Tests for scenarios where an external client interacts with Durable Task Agents. +/// +[Collection("Sequential")] +[Trait("Category", "Integration")] +public sealed class AgentEntityTests(ITestOutputHelper outputHelper) : IDisposable +{ + private static readonly TimeSpan s_defaultTimeout = Debugger.IsAttached + ? TimeSpan.FromMinutes(5) + : TimeSpan.FromSeconds(30); + + private static readonly IConfiguration s_configuration = + new ConfigurationBuilder() + .AddUserSecrets(Assembly.GetExecutingAssembly()) + .AddEnvironmentVariables() + .Build(); + + private readonly ITestOutputHelper _outputHelper = outputHelper; + private readonly CancellationTokenSource _cts = new(delay: s_defaultTimeout); + + private CancellationToken TestTimeoutToken => this._cts.Token; + + public void Dispose() => this._cts.Dispose(); + + [Fact] + public async Task EntityNamePrefixAsync() + { + // Setup + AIAgent simpleAgent = TestHelper.GetAzureOpenAIChatClient(s_configuration).CreateAIAgent( + name: "TestAgent", + instructions: "You are a helpful assistant that always responds with a friendly greeting." + ); + + using TestHelper testHelper = TestHelper.Start([simpleAgent], this._outputHelper); + + // A proxy agent is needed to call the hosted test agent + AIAgent simpleAgentProxy = simpleAgent.AsDurableAgentProxy(testHelper.Services); + + AgentThread thread = simpleAgentProxy.GetNewThread(); + + DurableTaskClient client = testHelper.GetClient(); + + AgentSessionId sessionId = thread.GetService(); + EntityInstanceId expectedEntityId = new($"dafx-{simpleAgent.Name}", sessionId.Key); + + EntityMetadata? entity = await client.Entities.GetEntityAsync(expectedEntityId, false, this.TestTimeoutToken); + + Assert.Null(entity); + + // Act: send a prompt to the agent + await simpleAgentProxy.RunAsync( + message: "Hello!", + thread, + cancellationToken: this.TestTimeoutToken); + + // Assert: verify the agent state was stored with the correct entity name prefix + entity = await client.Entities.GetEntityAsync(expectedEntityId, false, this.TestTimeoutToken); + + Assert.NotNull(entity); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/ExternalClientTests.cs b/dotnet/tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/ExternalClientTests.cs new file mode 100644 index 0000000000..241e05a843 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/ExternalClientTests.cs @@ -0,0 +1,215 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; +using System.Diagnostics; +using System.Reflection; +using Microsoft.Agents.AI.DurableTask.IntegrationTests.Logging; +using Microsoft.DurableTask; +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.Configuration; +using OpenAI; +using Xunit.Abstractions; + +namespace Microsoft.Agents.AI.DurableTask.IntegrationTests; + +/// +/// Tests for scenarios where an external client interacts with Durable Task Agents. +/// +[Collection("Sequential")] +[Trait("Category", "Integration")] +public sealed class ExternalClientTests(ITestOutputHelper outputHelper) : IDisposable +{ + private static readonly TimeSpan s_defaultTimeout = Debugger.IsAttached + ? TimeSpan.FromMinutes(5) + : TimeSpan.FromSeconds(30); + + private static readonly IConfiguration s_configuration = + new ConfigurationBuilder() + .AddUserSecrets(Assembly.GetExecutingAssembly()) + .AddEnvironmentVariables() + .Build(); + + private readonly ITestOutputHelper _outputHelper = outputHelper; + private readonly CancellationTokenSource _cts = new(delay: s_defaultTimeout); + + private CancellationToken TestTimeoutToken => this._cts.Token; + + public void Dispose() => this._cts.Dispose(); + + [Fact] + public async Task SimplePromptAsync() + { + // Setup + AIAgent simpleAgent = TestHelper.GetAzureOpenAIChatClient(s_configuration).CreateAIAgent( + instructions: "You are a helpful assistant that always responds with a friendly greeting.", + name: "TestAgent"); + + using TestHelper testHelper = TestHelper.Start([simpleAgent], this._outputHelper); + + // A proxy agent is needed to call the hosted test agent + AIAgent simpleAgentProxy = simpleAgent.AsDurableAgentProxy(testHelper.Services); + + // Act: send a prompt to the agent and wait for a response + AgentThread thread = simpleAgentProxy.GetNewThread(); + await simpleAgentProxy.RunAsync( + message: "Hello!", + thread, + cancellationToken: this.TestTimeoutToken); + + AgentRunResponse response = await simpleAgentProxy.RunAsync( + message: "Repeat what you just said but say it like a pirate", + thread, + cancellationToken: this.TestTimeoutToken); + + // Assert: verify the agent responded appropriately + // We can't predict the exact response, but we can check that there is one response + Assert.NotNull(response); + Assert.NotEmpty(response.Text); + + // Assert: verify the expected log entries were created in the expected category + IReadOnlyCollection logs = testHelper.GetLogs(); + Assert.NotEmpty(logs); + List agentLogs = [.. logs.Where(log => log.Category.Contains(simpleAgent.Name!)).ToList()]; + Assert.NotEmpty(agentLogs); + Assert.Contains(agentLogs, log => log.EventId.Name == "LogAgentRequest" && log.Message.Contains("Hello!")); + Assert.Contains(agentLogs, log => log.EventId.Name == "LogAgentResponse"); + } + + [Fact] + public async Task CallFunctionToolsAsync() + { + int weatherToolInvocationCount = 0; + int packingListToolInvocationCount = 0; + + string GetWeather(string location) + { + weatherToolInvocationCount++; + return $"The weather in {location} is sunny with a high of 75°F and a low of 55°F."; + } + + string SuggestPackingList(string weather, bool isSunny) + { + packingListToolInvocationCount++; + return isSunny ? "Pack sunglasses and sunscreen." : "Pack a raincoat and umbrella."; + } + + AIAgent tripPlanningAgent = TestHelper.GetAzureOpenAIChatClient(s_configuration).CreateAIAgent( + instructions: "You are a trip planning assistant. Use the weather tool and packing list tool as needed.", + name: "TripPlanningAgent", + description: "An agent to help plan your day trips", + tools: [AIFunctionFactory.Create(GetWeather), AIFunctionFactory.Create(SuggestPackingList)] + ); + + using TestHelper testHelper = TestHelper.Start([tripPlanningAgent], this._outputHelper); + AIAgent tripPlanningAgentProxy = tripPlanningAgent.AsDurableAgentProxy(testHelper.Services); + + // Act: send a prompt to the agent + AgentRunResponse response = await tripPlanningAgentProxy.RunAsync( + message: "Help me figure out what to pack for my Seattle trip next Sunday", + cancellationToken: this.TestTimeoutToken); + + // Assert: verify the agent responded appropriately + // We can't predict the exact response, but we can check that there is one response + Assert.NotNull(response); + Assert.NotEmpty(response.Text); + + // Assert: verify the expected log entries were created in the expected category + IReadOnlyCollection logs = testHelper.GetLogs(); + Assert.NotEmpty(logs); + + List agentLogs = [.. logs.Where(log => log.Category.Contains(tripPlanningAgent.Name!)).ToList()]; + Assert.NotEmpty(agentLogs); + Assert.Contains(agentLogs, log => log.EventId.Name == "LogAgentRequest" && log.Message.Contains("Seattle trip")); + Assert.Contains(agentLogs, log => log.EventId.Name == "LogAgentResponse"); + + // Assert: verify the tools were called + Assert.Equal(1, weatherToolInvocationCount); + Assert.Equal(1, packingListToolInvocationCount); + } + + [Fact] + public async Task CallLongRunningFunctionToolsAsync() + { + [Description("Starts a greeting workflow and returns the workflow instance ID")] + string StartWorkflowTool(string name) + { + return DurableAgentContext.Current.ScheduleNewOrchestration(nameof(RunWorkflowAsync), input: name); + } + + [Description("Gets the current status of a previously started workflow. A null response means the workflow has not started yet.")] + static async Task GetWorkflowStatusToolAsync(string instanceId) + { + OrchestrationMetadata? status = await DurableAgentContext.Current.GetOrchestrationStatusAsync( + instanceId, + includeDetails: true); + if (status == null) + { + // If the status is not found, wait a bit before returning null to give the workflow time to start + await Task.Delay(TimeSpan.FromSeconds(1)); + } + + return status; + } + + async Task RunWorkflowAsync(TaskOrchestrationContext context, string name) + { + // 1. Get agent and create a session + DurableAIAgent agent = context.GetAgent("SimpleAgent"); + AgentThread thread = agent.GetNewThread(); + + // 2. Call an agent and tell it my name + await agent.RunAsync($"My name is {name}.", thread); + + // 3. Call the agent again with the same thread (ask it to tell me my name) + AgentRunResponse response = await agent.RunAsync("What is my name?", thread); + + return response.Text; + } + + using TestHelper testHelper = TestHelper.Start( + this._outputHelper, + configureAgents: agents => + { + // This is the agent that will be used to start the workflow + agents.AddAIAgentFactory( + "WorkflowAgent", + sp => TestHelper.GetAzureOpenAIChatClient(s_configuration).CreateAIAgent( + name: "WorkflowAgent", + instructions: "You can start greeting workflows and check their status.", + services: sp, + tools: [ + AIFunctionFactory.Create(StartWorkflowTool), + AIFunctionFactory.Create(GetWorkflowStatusToolAsync) + ])); + + // This is the agent that will be called by the workflow + agents.AddAIAgent(TestHelper.GetAzureOpenAIChatClient(s_configuration).CreateAIAgent( + name: "SimpleAgent", + instructions: "You are a simple assistant." + )); + }, + durableTaskRegistry: registry => registry.AddOrchestratorFunc(nameof(RunWorkflowAsync), RunWorkflowAsync)); + + AIAgent workflowManagerAgentProxy = testHelper.Services.GetDurableAgentProxy("WorkflowAgent"); + + // Act: send a prompt to the agent + AgentThread thread = workflowManagerAgentProxy.GetNewThread(); + await workflowManagerAgentProxy.RunAsync( + message: "Start a greeting workflow for \"John Doe\".", + thread, + cancellationToken: this.TestTimeoutToken); + + // Act: prompt it again to wait for the workflow to complete + AgentRunResponse response = await workflowManagerAgentProxy.RunAsync( + message: "Wait for the workflow to complete and tell me the result.", + thread, + cancellationToken: this.TestTimeoutToken); + + // Assert: verify the agent responded appropriately + // We can't predict the exact response, but we can check that there is one response + Assert.NotNull(response); + Assert.NotEmpty(response.Text); + Assert.Contains("John Doe", response.Text); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/Logging/LogEntry.cs b/dotnet/tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/Logging/LogEntry.cs new file mode 100644 index 0000000000..fa9eddaeb4 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/Logging/LogEntry.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.Logging; + +namespace Microsoft.Agents.AI.DurableTask.IntegrationTests.Logging; + +internal sealed class LogEntry( + string category, + LogLevel level, + EventId eventId, + Exception? exception, + string message, + object? state, + IReadOnlyList> contextProperties) +{ + public string Category { get; } = category; + + public DateTime Timestamp { get; } = DateTime.Now; + + public EventId EventId { get; } = eventId; + + public LogLevel LogLevel { get; } = level; + + public Exception? Exception { get; } = exception; + + public string Message { get; } = message; + + public object? State { get; } = state; + + public IReadOnlyList> ContextProperties { get; } = contextProperties; + + public override string ToString() + { + string properties = this.ContextProperties.Count > 0 + ? $"[{string.Join(", ", this.ContextProperties.Select(kvp => $"{kvp.Key}={kvp.Value}"))}] " + : string.Empty; + + string eventName = this.EventId.Name ?? string.Empty; + string output = $"{this.Timestamp:o} [{this.Category}] {eventName} {properties}{this.Message}"; + + if (this.Exception is not null) + { + output += Environment.NewLine + this.Exception; + } + + return output; + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/Logging/TestLogger.cs b/dotnet/tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/Logging/TestLogger.cs new file mode 100644 index 0000000000..ca80b8cf7b --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/Logging/TestLogger.cs @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; +using Xunit.Abstractions; + +namespace Microsoft.Agents.AI.DurableTask.IntegrationTests.Logging; + +internal sealed class TestLogger(string category, ITestOutputHelper output) : ILogger +{ + private readonly string _category = category; + private readonly ITestOutputHelper _output = output; + private readonly ConcurrentQueue _entries = new(); + + public IReadOnlyCollection GetLogs() => this._entries; + + public void ClearLogs() => this._entries.Clear(); + + IDisposable? ILogger.BeginScope(TState state) => null; + + bool ILogger.IsEnabled(LogLevel logLevel) => true; + + void ILogger.Log( + LogLevel logLevel, + EventId eventId, + TState state, + Exception? exception, + Func formatter) + { + LogEntry entry = new( + category: this._category, + level: logLevel, + eventId: eventId, + exception: exception, + message: formatter(state, exception), + state: state, + contextProperties: []); + + this._entries.Enqueue(entry); + + try + { + this._output.WriteLine(entry.ToString()); + } + catch (InvalidOperationException) + { + // Expected when tests are shutting down + } + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/Logging/TestLoggerProvider.cs b/dotnet/tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/Logging/TestLoggerProvider.cs new file mode 100644 index 0000000000..7019852e5e --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/Logging/TestLoggerProvider.cs @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Concurrent; +using Microsoft.Extensions.Logging; +using Xunit.Abstractions; + +namespace Microsoft.Agents.AI.DurableTask.IntegrationTests.Logging; + +internal sealed class TestLoggerProvider(ITestOutputHelper output) : ILoggerProvider +{ + private readonly ITestOutputHelper _output = output ?? throw new ArgumentNullException(nameof(output)); + private readonly ConcurrentDictionary _loggers = new(StringComparer.OrdinalIgnoreCase); + + public bool TryGetLogs(string category, out IReadOnlyCollection logs) + { + if (this._loggers.TryGetValue(category, out TestLogger? logger)) + { + logs = logger.GetLogs(); + return true; + } + + logs = []; + return false; + } + + public IReadOnlyCollection GetAllLogs() + { + return this._loggers.Values + .OfType() + .SelectMany(logger => logger.GetLogs()) + .ToList() + .AsReadOnly(); + } + + public void Clear() + { + foreach (TestLogger logger in this._loggers.Values.OfType()) + { + logger.ClearLogs(); + } + } + + ILogger ILoggerProvider.CreateLogger(string categoryName) + { + return this._loggers.GetOrAdd(categoryName, _ => new TestLogger(categoryName, this._output)); + } + + void IDisposable.Dispose() + { + // no-op + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/Microsoft.Agents.AI.DurableTask.IntegrationTests.csproj b/dotnet/tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/Microsoft.Agents.AI.DurableTask.IntegrationTests.csproj new file mode 100644 index 0000000000..7150e74bd8 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/Microsoft.Agents.AI.DurableTask.IntegrationTests.csproj @@ -0,0 +1,23 @@ + + + + $(ProjectsCoreTargetFrameworks) + $(ProjectsDebugCoreTargetFrameworks) + enable + b7762d10-e29b-4bb1-8b74-b6d69a667dd4 + + + + + + + + + + + + + + + + diff --git a/dotnet/tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/TestHelper.cs b/dotnet/tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/TestHelper.cs new file mode 100644 index 0000000000..53c00c91ed --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.DurableTask.IntegrationTests/TestHelper.cs @@ -0,0 +1,138 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Azure; +using Azure.AI.OpenAI; +using Azure.Identity; +using Microsoft.Agents.AI.DurableTask.IntegrationTests.Logging; +using Microsoft.DurableTask; +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.AzureManaged; +using Microsoft.DurableTask.Worker; +using Microsoft.DurableTask.Worker.AzureManaged; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using OpenAI.Chat; +using Xunit.Abstractions; + +namespace Microsoft.Agents.AI.DurableTask.IntegrationTests; + +internal sealed class TestHelper : IDisposable +{ + private readonly TestLoggerProvider _loggerProvider; + private readonly IHost _host; + private readonly DurableTaskClient _client; + + // The static Start method should be used to create instances of this class. + private TestHelper( + TestLoggerProvider loggerProvider, + IHost host, + DurableTaskClient client) + { + this._loggerProvider = loggerProvider; + this._host = host; + this._client = client; + } + + public IServiceProvider Services => this._host.Services; + + public void Dispose() + { + this._host.Dispose(); + } + + public bool TryGetLogs(string category, out IReadOnlyCollection logs) + => this._loggerProvider.TryGetLogs(category, out logs); + + public static TestHelper Start( + AIAgent[] agents, + ITestOutputHelper outputHelper, + Action? durableTaskRegistry = null) + { + return BuildAndStartTestHelper( + outputHelper, + options => options.AddAIAgents(agents), + durableTaskRegistry); + } + + public static TestHelper Start( + ITestOutputHelper outputHelper, + Action configureAgents, + Action? durableTaskRegistry = null) + { + return BuildAndStartTestHelper( + outputHelper, + configureAgents, + durableTaskRegistry); + } + + public DurableTaskClient GetClient() => this._client; + + private static TestHelper BuildAndStartTestHelper( + ITestOutputHelper outputHelper, + Action configureAgents, + Action? durableTaskRegistry) + { + TestLoggerProvider loggerProvider = new(outputHelper); + + IHost host = Host.CreateDefaultBuilder() + .ConfigureServices((ctx, services) => + { + string dtsConnectionString = GetDurableTaskSchedulerConnectionString(ctx.Configuration); + + // Register durable agents using the caller-supplied registration action and + // apply the default chat client for agents that don't supply one themselves. + services.ConfigureDurableAgents( + options => configureAgents(options), + workerBuilder: builder => + { + builder.UseDurableTaskScheduler(dtsConnectionString); + if (durableTaskRegistry != null) + { + builder.AddTasks(durableTaskRegistry); + } + }, + clientBuilder: builder => builder.UseDurableTaskScheduler(dtsConnectionString)); + }) + .ConfigureLogging((_, logging) => + { + logging.AddProvider(loggerProvider); + logging.SetMinimumLevel(LogLevel.Debug); + }) + .Build(); + host.Start(); + + DurableTaskClient client = host.Services.GetRequiredService(); + return new TestHelper(loggerProvider, host, client); + } + + private static string GetDurableTaskSchedulerConnectionString(IConfiguration configuration) + { + // The default value is for local development using the Durable Task Scheduler emulator. + return configuration["DURABLE_TASK_SCHEDULER_CONNECTION_STRING"] + ?? "Endpoint=http://localhost:8080;TaskHub=default;Authentication=None"; + } + + internal static ChatClient GetAzureOpenAIChatClient(IConfiguration configuration) + { + string azureOpenAiEndpoint = configuration["AZURE_OPENAI_ENDPOINT"] + ?? throw new InvalidOperationException("The required AZURE_OPENAI_ENDPOINT env variable is not set."); + string azureOpenAiDeploymentName = configuration["AZURE_OPENAI_DEPLOYMENT"] + ?? throw new InvalidOperationException("The required AZURE_OPENAI_DEPLOYMENT env variable is not set."); + + // Check if AZURE_OPENAI_KEY is provided for token-based authentication + string? azureOpenAiKey = configuration["AZURE_OPENAI_KEY"]; + + AzureOpenAIClient client = !string.IsNullOrEmpty(azureOpenAiKey) + ? new AzureOpenAIClient(new Uri(azureOpenAiEndpoint), new AzureKeyCredential(azureOpenAiKey)) + : new AzureOpenAIClient(new Uri(azureOpenAiEndpoint), new AzureCliCredential()); + + return client.GetChatClient(azureOpenAiDeploymentName); + } + + internal IReadOnlyCollection GetLogs() + { + return this._loggerProvider.GetAllLogs(); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.DurableTask.UnitTests/Microsoft.Agents.AI.DurableTask.UnitTests.csproj b/dotnet/tests/Microsoft.Agents.AI.DurableTask.UnitTests/Microsoft.Agents.AI.DurableTask.UnitTests.csproj index 02ed1b58e7..b413733f2b 100644 --- a/dotnet/tests/Microsoft.Agents.AI.DurableTask.UnitTests/Microsoft.Agents.AI.DurableTask.UnitTests.csproj +++ b/dotnet/tests/Microsoft.Agents.AI.DurableTask.UnitTests/Microsoft.Agents.AI.DurableTask.UnitTests.csproj @@ -4,6 +4,7 @@ $(ProjectsCoreTargetFrameworks) $(ProjectsDebugCoreTargetFrameworks) enable + b7762d10-e29b-4bb1-8b74-b6d69a667dd4 diff --git a/dotnet/tests/Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests/Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests.csproj b/dotnet/tests/Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests/Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests.csproj new file mode 100644 index 0000000000..ae816efb7f --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests/Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests.csproj @@ -0,0 +1,19 @@ + + + + $(ProjectsCoreTargetFrameworks) + $(ProjectsDebugCoreTargetFrameworks) + enable + b7762d10-e29b-4bb1-8b74-b6d69a667dd4 + + + + + + + + + + + + diff --git a/dotnet/tests/Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests/SamplesValidation.cs b/dotnet/tests/Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests/SamplesValidation.cs new file mode 100644 index 0000000000..9572b097ae --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests/SamplesValidation.cs @@ -0,0 +1,816 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics; +using System.Reflection; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using ModelContextProtocol.Client; +using ModelContextProtocol.Protocol; +using Xunit.Abstractions; + +namespace Microsoft.Agents.AI.Hosting.AzureFunctions.IntegrationTests; + +[Collection("Samples")] +[Trait("Category", "SampleValidation")] +public sealed class SamplesValidation(ITestOutputHelper outputHelper) : IAsyncLifetime +{ + private const string AzureFunctionsPort = "7071"; + private const string AzuritePort = "10000"; + private const string DtsPort = "8080"; + + private static readonly string s_dotnetTargetFramework = GetTargetFramework(); + private static readonly HttpClient s_sharedHttpClient = new(); + private static readonly IConfiguration s_configuration = + new ConfigurationBuilder() + .AddUserSecrets(Assembly.GetExecutingAssembly()) + .AddEnvironmentVariables() + .Build(); + + private static bool s_infrastructureStarted; + private static readonly TimeSpan s_orchestrationTimeout = TimeSpan.FromMinutes(1); + private static readonly string s_samplesPath = Path.GetFullPath( + Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..", "..", "..", "..", "..", "samples", "AzureFunctions")); + + private readonly ITestOutputHelper _outputHelper = outputHelper; + + async Task IAsyncLifetime.InitializeAsync() + { + if (!s_infrastructureStarted) + { + await this.StartSharedInfrastructureAsync(); + s_infrastructureStarted = true; + } + } + + async Task IAsyncLifetime.DisposeAsync() + { + // Nothing to clean up + await Task.CompletedTask; + } + + [Fact] + public async Task SingleAgentSampleValidationAsync() + { + string samplePath = Path.Combine(s_samplesPath, "01_SingleAgent"); + await this.RunSampleTestAsync(samplePath, async (logs) => + { + Uri startUri = new($"http://localhost:{AzureFunctionsPort}/api/agents/Joker/run"); + this._outputHelper.WriteLine($"Starting single agent orchestration via POST request to {startUri}..."); + + // Test the agent endpoint as described in the README + const string RequestBody = "Tell me a joke about a pirate."; + using HttpContent content = new StringContent(RequestBody, Encoding.UTF8, "text/plain"); + + using HttpResponseMessage response = await s_sharedHttpClient.PostAsync(startUri, content); + + // The response is expected to be a plain text response with the agent's reply (the joke) + Assert.True(response.IsSuccessStatusCode, $"Agent request failed with status: {response.StatusCode}"); + Assert.Equal("text/plain", response.Content.Headers.ContentType?.MediaType); + string responseText = await response.Content.ReadAsStringAsync(); + Assert.NotEmpty(responseText); + this._outputHelper.WriteLine($"Agent run response: {responseText}"); + + // The response headers should include the agent thread ID, which can be used to continue the conversation. + string? threadId = response.Headers.GetValues("X-Agent-Thread")?.FirstOrDefault(); + Assert.NotNull(threadId); + + this._outputHelper.WriteLine($"Agent thread ID: {threadId}"); + Assert.StartsWith("@dafx-joker@", threadId); + + // Wait for up to 30 seconds to see if the agent response is available in the logs + await this.WaitForConditionAsync( + condition: () => + { + lock (logs) + { + bool exists = logs.Any( + log => log.Message.Contains("Response:") && log.Message.Contains(threadId)); + return Task.FromResult(exists); + } + }, + message: "Agent response is available", + timeout: TimeSpan.FromSeconds(30)); + }); + } + + [Fact] + public async Task SingleAgentOrchestrationChainingSampleValidationAsync() + { + string samplePath = Path.Combine(s_samplesPath, "02_AgentOrchestration_Chaining"); + await this.RunSampleTestAsync(samplePath, async (logs) => + { + Uri startUri = new($"http://localhost:{AzureFunctionsPort}/api/singleagent/run"); + this._outputHelper.WriteLine($"Starting single agent orchestration via POST request to {startUri}..."); + + // Start the orchestration + using HttpResponseMessage startResponse = await s_sharedHttpClient.PostAsync(startUri, content: null); + + Assert.True( + startResponse.IsSuccessStatusCode, + $"Start orchestration failed with status: {startResponse.StatusCode}"); + string startResponseText = await startResponse.Content.ReadAsStringAsync(); + JsonElement startResult = JsonSerializer.Deserialize(startResponseText); + + Assert.True(startResult.TryGetProperty("statusQueryGetUri", out JsonElement statusUriElement)); + Uri statusUri = new(statusUriElement.GetString()!); + + // Wait for orchestration to complete + await this.WaitForOrchestrationCompletionAsync(statusUri); + + // Verify the final result + using HttpResponseMessage statusResponse = await s_sharedHttpClient.GetAsync(statusUri); + Assert.True( + statusResponse.IsSuccessStatusCode, + $"Status check failed with status: {statusResponse.StatusCode}"); + + string statusText = await statusResponse.Content.ReadAsStringAsync(); + JsonElement statusResult = JsonSerializer.Deserialize(statusText); + + Assert.Equal("Completed", statusResult.GetProperty("runtimeStatus").GetString()); + Assert.True(statusResult.TryGetProperty("output", out JsonElement outputElement)); + string? output = outputElement.GetString(); + + // Can't really validate the output since it's non-deterministic, but we can at least check it's non-empty + Assert.NotNull(output); + Assert.True(output.Length > 20, "Output is unexpectedly short"); + }); + } + + [Fact] + public async Task MultiAgentOrchestrationConcurrentSampleValidationAsync() + { + string samplePath = Path.Combine(s_samplesPath, "03_AgentOrchestration_Concurrency"); + await this.RunSampleTestAsync(samplePath, async (logs) => + { + // Start the multi-agent orchestration + const string RequestBody = "What is temperature?"; + using HttpContent content = new StringContent(RequestBody, Encoding.UTF8, "text/plain"); + + Uri startUri = new($"http://localhost:{AzureFunctionsPort}/api/multiagent/run"); + this._outputHelper.WriteLine($"Starting multi agent orchestration via POST request to {startUri}..."); + using HttpResponseMessage startResponse = await s_sharedHttpClient.PostAsync(startUri, content); + + Assert.True(startResponse.IsSuccessStatusCode, $"Start orchestration failed with status: {startResponse.StatusCode}"); + string startResponseText = await startResponse.Content.ReadAsStringAsync(); + JsonElement startResult = JsonSerializer.Deserialize(startResponseText); + + Assert.True(startResult.TryGetProperty("instanceId", out JsonElement instanceIdElement)); + Assert.True(startResult.TryGetProperty("statusQueryGetUri", out JsonElement statusUriElement)); + + Uri statusUri = new(statusUriElement.GetString()!); + + // Wait for orchestration to complete + await this.WaitForOrchestrationCompletionAsync(statusUri); + + // Verify the final result + using HttpResponseMessage statusResponse = await s_sharedHttpClient.GetAsync(statusUri); + Assert.True(statusResponse.IsSuccessStatusCode, $"Status check failed with status: {statusResponse.StatusCode}"); + + string statusText = await statusResponse.Content.ReadAsStringAsync(); + JsonElement statusResult = JsonSerializer.Deserialize(statusText); + + Assert.Equal("Completed", statusResult.GetProperty("runtimeStatus").GetString()); + Assert.True(statusResult.TryGetProperty("output", out JsonElement outputElement)); + + // Verify both physicist and chemist responses are present + Assert.True(outputElement.TryGetProperty("physicist", out JsonElement physicistElement)); + Assert.True(outputElement.TryGetProperty("chemist", out JsonElement chemistElement)); + + string physicistResponse = physicistElement.GetString()!; + string chemistResponse = chemistElement.GetString()!; + + Assert.NotEmpty(physicistResponse); + Assert.NotEmpty(chemistResponse); + Assert.Contains("temperature", physicistResponse, StringComparison.OrdinalIgnoreCase); + Assert.Contains("temperature", chemistResponse, StringComparison.OrdinalIgnoreCase); + }); + } + + [Fact] + public async Task MultiAgentOrchestrationConditionalsSampleValidationAsync() + { + string samplePath = Path.Combine(s_samplesPath, "04_AgentOrchestration_Conditionals"); + await this.RunSampleTestAsync(samplePath, async (logs) => + { + // Test with legitimate email + await this.TestSpamDetectionAsync("email-001", + "Hi John, I hope you're doing well. I wanted to follow up on our meeting yesterday about the quarterly report. Could you please send me the updated figures by Friday? Thanks!", + expectedSpam: false); + + // Test with spam email + await this.TestSpamDetectionAsync("email-002", + "URGENT! You've won $1,000,000! Click here now to claim your prize! Limited time offer! Don't miss out!", + expectedSpam: true); + }); + } + + [Fact] + public async Task SingleAgentOrchestrationHITLSampleValidationAsync() + { + string samplePath = Path.Combine(s_samplesPath, "05_AgentOrchestration_HITL"); + + await this.RunSampleTestAsync(samplePath, async (logs) => + { + // Start the HITL orchestration with short timeout for testing + // TODO: Add validation for the approval case + object requestBody = new + { + topic = "The Future of Artificial Intelligence", + max_review_attempts = 3, + approval_timeout_hours = 0.001 // Very short timeout for testing + }; + + string jsonContent = JsonSerializer.Serialize(requestBody); + using HttpContent content = new StringContent(jsonContent, Encoding.UTF8, "application/json"); + + Uri startUri = new($"http://localhost:{AzureFunctionsPort}/api/hitl/run"); + this._outputHelper.WriteLine($"Starting HITL orchestration via POST request to {startUri}..."); + using HttpResponseMessage startResponse = await s_sharedHttpClient.PostAsync(startUri, content); + + Assert.True( + startResponse.IsSuccessStatusCode, + $"Start HITL orchestration failed with status: {startResponse.StatusCode}"); + string startResponseText = await startResponse.Content.ReadAsStringAsync(); + JsonElement startResult = JsonSerializer.Deserialize(startResponseText); + + Assert.True(startResult.TryGetProperty("statusQueryGetUri", out JsonElement statusUriElement)); + Uri statusUri = new(statusUriElement.GetString()!); + + // Wait for orchestration to complete (it should timeout due to short timeout) + await this.WaitForOrchestrationCompletionAsync(statusUri); + + // Verify the final result + using HttpResponseMessage statusResponse = await s_sharedHttpClient.GetAsync(statusUri); + Assert.True( + statusResponse.IsSuccessStatusCode, + $"Status check failed with status: {statusResponse.StatusCode}"); + + string statusText = await statusResponse.Content.ReadAsStringAsync(); + this._outputHelper.WriteLine($"HITL orchestration status text: {statusText}"); + + JsonElement statusResult = JsonSerializer.Deserialize(statusText); + + // The orchestration should complete with a failed status due to timeout + Assert.Equal("Failed", statusResult.GetProperty("runtimeStatus").GetString()); + Assert.True(statusResult.TryGetProperty("failureDetails", out JsonElement failureDetailsElement)); + Assert.True(failureDetailsElement.TryGetProperty("ErrorType", out JsonElement errorTypeElement)); + Assert.Equal("System.TimeoutException", errorTypeElement.GetString()); + Assert.True(failureDetailsElement.TryGetProperty("ErrorMessage", out JsonElement errorMessageElement)); + Assert.StartsWith("Human approval timed out", errorMessageElement.GetString()); + }); + } + + [Fact] + public async Task LongRunningToolsSampleValidationAsync() + { + string samplePath = Path.Combine(s_samplesPath, "06_LongRunningTools"); + + await this.RunSampleTestAsync(samplePath, async (logs) => + { + // Test starting an agent that schedules a content generation orchestration + const string Prompt = "Start a content generation workflow for the topic 'The Future of Artificial Intelligence'"; + using HttpContent messageContent = new StringContent(Prompt, Encoding.UTF8, "text/plain"); + + Uri runAgentUri = new($"http://localhost:{AzureFunctionsPort}/api/agents/publisher/run"); + + this._outputHelper.WriteLine($"Starting agent tool orchestration via POST request to {runAgentUri}..."); + using HttpResponseMessage startResponse = await s_sharedHttpClient.PostAsync(runAgentUri, messageContent); + + Assert.True( + startResponse.IsSuccessStatusCode, + $"Start agent request failed with status: {startResponse.StatusCode}"); + + string startResponseText = await startResponse.Content.ReadAsStringAsync(); + this._outputHelper.WriteLine($"Agent response: {startResponseText}"); + + // The response should be deserializable as an AgentRunResponse object and have a valid thread ID + startResponse.Headers.TryGetValues("X-Agent-Thread", out IEnumerable? agentIdValues); + string? threadId = agentIdValues?.FirstOrDefault(); + Assert.NotNull(threadId); + Assert.StartsWith("@dafx-publisher@", threadId); + + // Wait for the orchestration to report that it's waiting for human approval + await this.WaitForConditionAsync( + condition: () => + { + // For now, we have to rely on the logs to check for the "NOTIFICATION" message that gets generated by the activity function. + // TODO: Synchronously prompt the agent for status + lock (logs) + { + bool exists = logs.Any(log => log.Message.Contains("NOTIFICATION: Please review the following content for approval")); + return Task.FromResult(exists); + } + }, + message: "Orchestration is requesting human feedback", + timeout: TimeSpan.FromSeconds(60)); + + // Approve the content + Uri approvalUri = new($"{runAgentUri}?threadId={threadId}"); + using HttpContent approvalContent = new StringContent("Approve the content", Encoding.UTF8, "text/plain"); + using HttpResponseMessage approvalResponse = await s_sharedHttpClient.PostAsync(approvalUri, approvalContent); + Assert.True(approvalResponse.IsSuccessStatusCode, $"Approve content request failed with status: {approvalResponse.StatusCode}"); + + // Wait for the publish notification to be logged + await this.WaitForConditionAsync( + condition: () => + { + lock (logs) + { + // TODO: Synchronously prompt the agent for status + bool exists = logs.Any(log => log.Message.Contains("PUBLISHING: Content has been published successfully")); + return Task.FromResult(exists); + } + }, + message: "Content published notification is logged", + timeout: TimeSpan.FromSeconds(60)); + + // Verify the final orchestration status by asking the agent for the status + Uri statusUri = new($"{runAgentUri}?threadId={threadId}"); + await this.WaitForConditionAsync( + condition: async () => + { + this._outputHelper.WriteLine($"Checking status of orchestration at {statusUri}..."); + + using StringContent content = new("Get the status of the workflow", Encoding.UTF8, "text/plain"); + using HttpResponseMessage statusResponse = await s_sharedHttpClient.PostAsync(statusUri, content); + Assert.True( + statusResponse.IsSuccessStatusCode, + $"Status check failed with status: {statusResponse.StatusCode}"); + string statusText = await statusResponse.Content.ReadAsStringAsync(); + this._outputHelper.WriteLine($"Status text: {statusText}"); + + bool isCompleted = statusText.Contains("Completed", StringComparison.OrdinalIgnoreCase); + bool hasContent = statusText.Contains( + "The Future of Artificial Intelligence", + StringComparison.OrdinalIgnoreCase); + return isCompleted && hasContent; + }, + message: "Orchestration is completed", + timeout: TimeSpan.FromSeconds(60)); + }); + } + + [Fact] + public async Task AgentAsMcpToolAsync() + { + string samplePath = Path.Combine(s_samplesPath, "07_AgentAsMcpTool"); + await this.RunSampleTestAsync(samplePath, async (logs) => + { + IClientTransport clientTransport = new HttpClientTransport(new() + { + Endpoint = new Uri($"http://localhost:{AzureFunctionsPort}/runtime/webhooks/mcp") + }); + + await using McpClient mcpClient = await McpClient.CreateAsync(clientTransport!); + + // Ensure the expected tools are present. + IList tools = await mcpClient.ListToolsAsync(); + + Assert.Single(tools, t => t.Name == "StockAdvisor"); + Assert.Single(tools, t => t.Name == "PlantAdvisor"); + + // Invoke the tools to verify they work as expected. + string stockPriceResponse = await this.InvokeMcpToolAsync(mcpClient, "StockAdvisor", "MSFT ATH"); + string plantSuggestionResponse = await this.InvokeMcpToolAsync(mcpClient, "PlantAdvisor", "Low light plant"); + Assert.NotEmpty(stockPriceResponse); + Assert.NotEmpty(plantSuggestionResponse); + + // Wait for up to 30 seconds to see if the agent responses are available in the logs + await this.WaitForConditionAsync( + condition: () => + { + lock (logs) + { + bool expectedLogsPresent = logs.Count(log => log.Message.Contains("Response:")) >= 2; + return Task.FromResult(expectedLogsPresent); + } + }, + message: "Agent response is available", + timeout: TimeSpan.FromSeconds(30)); + }); + } + + private async Task InvokeMcpToolAsync(McpClient mcpClient, string toolName, string query) + { + this._outputHelper.WriteLine($"Invoking MCP tool '{toolName}'..."); + + CallToolResult result = await mcpClient.CallToolAsync( + toolName, + arguments: new Dictionary { { "query", query } }); + + string toolCallResult = ((TextContentBlock)result.Content[0]).Text; + this._outputHelper.WriteLine($"MCP tool '{toolName}' response: {toolCallResult}"); + + return toolCallResult; + } + + private async Task TestSpamDetectionAsync(string emailId, string emailContent, bool expectedSpam) + { + object requestBody = new + { + email_id = emailId, + email_content = emailContent + }; + + string jsonContent = JsonSerializer.Serialize(requestBody); + using HttpContent content = new StringContent(jsonContent, Encoding.UTF8, "application/json"); + + Uri startUri = new($"http://localhost:{AzureFunctionsPort}/api/spamdetection/run"); + this._outputHelper.WriteLine($"Starting spam detection orchestration via POST request to {startUri}..."); + using HttpResponseMessage startResponse = await s_sharedHttpClient.PostAsync(startUri, content); + + Assert.True(startResponse.IsSuccessStatusCode, $"Start orchestration failed with status: {startResponse.StatusCode}"); + string startResponseText = await startResponse.Content.ReadAsStringAsync(); + JsonElement startResult = JsonSerializer.Deserialize(startResponseText); + + Assert.True(startResult.TryGetProperty("statusQueryGetUri", out JsonElement statusUriElement)); + Uri statusUri = new(statusUriElement.GetString()!); + + // Wait for orchestration to complete + await this.WaitForOrchestrationCompletionAsync(statusUri); + + // Verify the final result + using HttpResponseMessage statusResponse = await s_sharedHttpClient.GetAsync(statusUri); + Assert.True(statusResponse.IsSuccessStatusCode, $"Status check failed with status: {statusResponse.StatusCode}"); + + string statusText = await statusResponse.Content.ReadAsStringAsync(); + JsonElement statusResult = JsonSerializer.Deserialize(statusText); + + Assert.Equal("Completed", statusResult.GetProperty("runtimeStatus").GetString()); + Assert.True(statusResult.TryGetProperty("output", out JsonElement outputElement)); + + string output = outputElement.GetString()!; + Assert.NotEmpty(output); + + if (expectedSpam) + { + Assert.Contains("spam", output, StringComparison.OrdinalIgnoreCase); + } + else + { + Assert.Contains("sent", output, StringComparison.OrdinalIgnoreCase); + } + } + + private async Task StartSharedInfrastructureAsync() + { + // Start Azurite if it's not already running + if (!await this.IsAzuriteRunningAsync()) + { + await this.StartDockerContainerAsync( + containerName: "azurite", + image: "mcr.microsoft.com/azure-storage/azurite", + ports: ["-p", "10000:10000", "-p", "10001:10001", "-p", "10002:10002"]); + + // Wait for Azurite + await this.WaitForConditionAsync(this.IsAzuriteRunningAsync, "Azurite is running", TimeSpan.FromSeconds(30)); + } + + // Start DTS emulator if it's not already running + if (!await this.IsDtsEmulatorRunningAsync()) + { + await this.StartDockerContainerAsync( + containerName: "dts-emulator", + image: "mcr.microsoft.com/dts/dts-emulator:latest", + ports: ["-p", "8080:8080", "-p", "8082:8082"]); + + // Wait for DTS emulator + await this.WaitForConditionAsync( + condition: this.IsDtsEmulatorRunningAsync, + message: "DTS emulator is running", + timeout: TimeSpan.FromSeconds(30)); + } + } + + private async Task IsAzuriteRunningAsync() + { + this._outputHelper.WriteLine( + $"Checking if Azurite is running at http://localhost:{AzuritePort}/devstoreaccount1..."); + + try + { + using CancellationTokenSource timeoutCts = new(TimeSpan.FromSeconds(30)); + + // Example output when pinging Azurite: + // $ curl -i http://localhost:10000/devstoreaccount1?comp=list + // HTTP/1.1 403 Server failed to authenticate the request. + // Server: Azurite-Blob/3.34.0 + // x-ms-error-code: AuthorizationFailure + // x-ms-request-id: 6cd21522-bb0f-40f6-962c-fa174f17aa30 + // content-type: application/xml + // Date: Mon, 20 Oct 2025 23:52:02 GMT + // Connection: keep-alive + // Keep-Alive: timeout=5 + // Transfer-Encoding: chunked + using HttpResponseMessage response = await s_sharedHttpClient.GetAsync( + requestUri: new Uri($"http://localhost:{AzuritePort}/devstoreaccount1?comp=list"), + cancellationToken: timeoutCts.Token); + if (response.Headers.TryGetValues( + "Server", + out IEnumerable? serverValues) && serverValues.Any(s => s.StartsWith("Azurite", StringComparison.OrdinalIgnoreCase))) + { + this._outputHelper.WriteLine($"Azurite is running, server: {string.Join(", ", serverValues)}"); + return true; + } + + this._outputHelper.WriteLine($"Azurite is not running. Status code: {response.StatusCode}"); + return false; + } + catch (HttpRequestException ex) + { + this._outputHelper.WriteLine($"Azurite is not running: {ex.Message}"); + return false; + } + } + + private async Task IsDtsEmulatorRunningAsync() + { + this._outputHelper.WriteLine($"Checking if DTS emulator is running at http://localhost:{DtsPort}/healthz..."); + + // DTS emulator doesn't support HTTP/1.1, so we need to use HTTP/2.0 + using HttpClient http2Client = new() + { + DefaultRequestVersion = new Version(2, 0), + DefaultVersionPolicy = HttpVersionPolicy.RequestVersionExact + }; + + try + { + using CancellationTokenSource timeoutCts = new(TimeSpan.FromSeconds(30)); + using HttpResponseMessage response = await http2Client.GetAsync(new Uri($"http://localhost:{DtsPort}/healthz"), timeoutCts.Token); + if (response.Content.Headers.ContentLength > 0) + { + string content = await response.Content.ReadAsStringAsync(timeoutCts.Token); + this._outputHelper.WriteLine($"DTS emulator health check response: {content}"); + } + + if (response.IsSuccessStatusCode) + { + this._outputHelper.WriteLine("DTS emulator is running"); + return true; + } + + this._outputHelper.WriteLine($"DTS emulator is not running. Status code: {response.StatusCode}"); + return false; + } + catch (HttpRequestException ex) + { + this._outputHelper.WriteLine($"DTS emulator is not running: {ex.Message}"); + return false; + } + } + + private async Task StartDockerContainerAsync(string containerName, string image, string[] ports) + { + // Stop existing container if it exists + await this.RunCommandAsync("docker", ["stop", containerName]); + await this.RunCommandAsync("docker", ["rm", containerName]); + + // Start new container + List args = ["run", "-d", "--name", containerName]; + args.AddRange(ports); + args.Add(image); + + this._outputHelper.WriteLine( + $"Starting new container: {containerName} with image: {image} and ports: {string.Join(", ", ports)}"); + await this.RunCommandAsync("docker", args.ToArray()); + this._outputHelper.WriteLine($"Container started: {containerName}"); + } + + private async Task WaitForConditionAsync(Func> condition, string message, TimeSpan timeout) + { + this._outputHelper.WriteLine($"Waiting for '{message}'..."); + + using CancellationTokenSource cancellationTokenSource = new(timeout); + while (true) + { + if (await condition()) + { + return; + } + + try + { + await Task.Delay(TimeSpan.FromSeconds(1), cancellationTokenSource.Token); + } + catch (OperationCanceledException) when (cancellationTokenSource.IsCancellationRequested) + { + throw new TimeoutException($"Timeout waiting for '{message}'"); + } + } + } + + private async Task RunSampleTestAsync(string samplePath, Func, Task> testAction) + { + // Start the Azure Functions app + List logsContainer = []; + using Process funcProcess = this.StartFunctionApp(samplePath, logsContainer); + try + { + // Wait for the app to be ready + await this.WaitForAzureFunctionsAsync(); + + // Run the test + await testAction(logsContainer); + } + finally + { + await this.StopProcessAsync(funcProcess); + } + } + + private sealed record OutputLog(DateTime Timestamp, LogLevel Level, string Message); + + private Process StartFunctionApp(string samplePath, List logs) + { + ProcessStartInfo startInfo = new() + { + FileName = "dotnet", + Arguments = $"run -f {s_dotnetTargetFramework} --port {AzureFunctionsPort}", + WorkingDirectory = samplePath, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + }; + + string openAiEndpoint = s_configuration["AZURE_OPENAI_ENDPOINT"] + ?? throw new InvalidOperationException("AZURE_OPENAI_ENDPOINT is not set in the environment variables or user secrets"); + string openAiDeployment = s_configuration["AZURE_OPENAI_DEPLOYMENT"] + ?? throw new InvalidOperationException("AZURE_OPENAI_DEPLOYMENT is not set in the environment variables or user secrets"); + + // Set required environment variables for the function app (see local.settings.json for required settings) + startInfo.EnvironmentVariables["AZURE_OPENAI_ENDPOINT"] = openAiEndpoint; + startInfo.EnvironmentVariables["AZURE_OPENAI_DEPLOYMENT"] = openAiDeployment; + startInfo.EnvironmentVariables["DURABLE_TASK_SCHEDULER_CONNECTION_STRING"] = + $"Endpoint=http://localhost:{DtsPort};TaskHub=default;Authentication=None"; + startInfo.EnvironmentVariables["AzureWebJobsStorage"] = "UseDevelopmentStorage=true"; + + Process process = new() { StartInfo = startInfo }; + + // Capture the output and error streams + process.ErrorDataReceived += (sender, e) => + { + if (e.Data != null) + { + this._outputHelper.WriteLine($"[{startInfo.FileName}(err)]: {e.Data}"); + lock (logs) + { + logs.Add(new OutputLog(DateTime.Now, LogLevel.Error, e.Data)); + } + } + }; + + process.OutputDataReceived += (sender, e) => + { + if (e.Data != null) + { + this._outputHelper.WriteLine($"[{startInfo.FileName}(out)]: {e.Data}"); + lock (logs) + { + logs.Add(new OutputLog(DateTime.Now, LogLevel.Information, e.Data)); + } + } + }; + + if (!process.Start()) + { + throw new InvalidOperationException("Failed to start the function app"); + } + + process.BeginErrorReadLine(); + process.BeginOutputReadLine(); + + return process; + } + + private async Task WaitForAzureFunctionsAsync() + { + this._outputHelper.WriteLine( + $"Waiting for Azure Functions Core Tools to be ready at http://localhost:{AzureFunctionsPort}/..."); + await this.WaitForConditionAsync( + condition: async () => + { + try + { + using HttpRequestMessage request = new(HttpMethod.Head, $"http://localhost:{AzureFunctionsPort}/"); + using HttpResponseMessage response = await s_sharedHttpClient.SendAsync(request); + this._outputHelper.WriteLine($"Azure Functions Core Tools response: {response.StatusCode}"); + return response.IsSuccessStatusCode; + } + catch (HttpRequestException) + { + // Expected when the app isn't yet ready + return false; + } + }, + message: "Azure Functions Core Tools is ready", + timeout: TimeSpan.FromSeconds(60)); + } + + private async Task WaitForOrchestrationCompletionAsync(Uri statusUri) + { + using CancellationTokenSource timeoutCts = new(s_orchestrationTimeout); + while (true) + { + try + { + using HttpResponseMessage response = await s_sharedHttpClient.GetAsync( + statusUri, + timeoutCts.Token); + if (response.IsSuccessStatusCode) + { + string responseText = await response.Content.ReadAsStringAsync(timeoutCts.Token); + JsonElement result = JsonSerializer.Deserialize(responseText); + + if (result.TryGetProperty("runtimeStatus", out JsonElement statusElement)) + { + string status = statusElement.GetString()!; + if (status == "Completed" || status == "Failed" || status == "Terminated") + { + return; + } + } + } + } + catch (Exception ex) when (!timeoutCts.Token.IsCancellationRequested) + { + // Ignore errors and retry + this._outputHelper.WriteLine($"Error waiting for orchestration completion: {ex}"); + } + + await Task.Delay(TimeSpan.FromSeconds(1), timeoutCts.Token); + } + } + + private async Task RunCommandAsync(string command, string[] args) + { + await this.RunCommandAsync(command, workingDirectory: null, args: args); + } + + private async Task RunCommandAsync(string command, string? workingDirectory, string[] args) + { + ProcessStartInfo startInfo = new() + { + FileName = command, + Arguments = string.Join(" ", args), + WorkingDirectory = workingDirectory, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true + }; + + this._outputHelper.WriteLine($"Running command: {command} {string.Join(" ", args)}"); + + using Process process = new() { StartInfo = startInfo }; + process.ErrorDataReceived += (sender, e) => this._outputHelper.WriteLine($"[{command}(err)]: {e.Data}"); + process.OutputDataReceived += (sender, e) => this._outputHelper.WriteLine($"[{command}(out)]: {e.Data}"); + if (!process.Start()) + { + throw new InvalidOperationException("Failed to start the command"); + } + process.BeginErrorReadLine(); + process.BeginOutputReadLine(); + + using CancellationTokenSource cancellationTokenSource = new(TimeSpan.FromMinutes(1)); + await process.WaitForExitAsync(cancellationTokenSource.Token); + + this._outputHelper.WriteLine($"Command completed with exit code: {process.ExitCode}"); + } + + private async Task StopProcessAsync(Process process) + { + try + { + if (!process.HasExited) + { + this._outputHelper.WriteLine($"Killing process {process.ProcessName}#{process.Id}"); + process.Kill(entireProcessTree: true); + + using CancellationTokenSource timeoutCts = new(TimeSpan.FromSeconds(10)); + await process.WaitForExitAsync(timeoutCts.Token); + this._outputHelper.WriteLine($"Process exited: {process.Id}"); + } + } + catch (Exception ex) + { + this._outputHelper.WriteLine($"Failed to stop process: {ex.Message}"); + } + } + + private static string GetTargetFramework() + { + // Get the target framework by looking at the path of the current file. It should be something like /path/to/project/bin/Debug/net8.0/... + string filePath = new Uri(typeof(SamplesValidation).Assembly.Location).LocalPath; + string directory = Path.GetDirectoryName(filePath)!; + string tfm = Path.GetFileName(directory); + if (tfm.StartsWith("net", StringComparison.OrdinalIgnoreCase)) + { + return tfm; + } + + throw new InvalidOperationException($"Unable to find target framework in path: {filePath}"); + } +}