-
Notifications
You must be signed in to change notification settings - Fork 3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
.Net: Integration Tests for Function Calling Stepwise Planner (#4189)
### Motivation and Context / Description <!-- Thank you for your contribution to the semantic-kernel repo! Please help reviewers and future users, providing the following information: 1. Why is this change required? 2. What problem does it solve? 3. What scenario does it contribute to? 4. If it fixes an open issue, please link to the issue here. --> - The integration tests for Function Calling Stepwise Planner were removed when we refactored/removed `Planners.Core`. This adds them back and improves them with more test cases and results validation. - Also includes a minor fix in sample code (`ChatHistory.AsJson()` no longer exists) ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [x] The code builds clean without any errors or warnings - [x] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [x] All unit tests pass, and I have added new tests where possible - [x] I didn't break anyone 😄
- Loading branch information
Showing
3 changed files
with
114 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
112 changes: 112 additions & 0 deletions
112
dotnet/src/IntegrationTests/Planners/Stepwise/FunctionCallingStepwisePlannerTests.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,112 @@ | ||
// Copyright (c) Microsoft. All rights reserved. | ||
|
||
using System; | ||
using System.Threading.Tasks; | ||
using Microsoft.Extensions.Configuration; | ||
using Microsoft.SemanticKernel.Planning; | ||
using Microsoft.SemanticKernel.Plugins.Core; | ||
using Microsoft.SemanticKernel.Plugins.Web.Bing; | ||
using Microsoft.SemanticKernel.Plugins.Web; | ||
using Microsoft.SemanticKernel; | ||
using SemanticKernel.IntegrationTests.TestSettings; | ||
using Xunit.Abstractions; | ||
using Xunit; | ||
using SemanticKernel.IntegrationTests.Fakes; | ||
using System.Text.Json; | ||
|
||
namespace SemanticKernel.IntegrationTests.Planners.Stepwise; | ||
public sealed class FunctionCallingStepwisePlannerTests : IDisposable | ||
{ | ||
private readonly string _bingApiKey; | ||
|
||
public FunctionCallingStepwisePlannerTests(ITestOutputHelper output) | ||
{ | ||
this._testOutputHelper = new RedirectOutput(output); | ||
|
||
// Load configuration | ||
this._configuration = new ConfigurationBuilder() | ||
.AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) | ||
.AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) | ||
.AddEnvironmentVariables() | ||
.AddUserSecrets<FunctionCallingStepwisePlannerTests>() | ||
.Build(); | ||
|
||
string? bingApiKeyCandidate = this._configuration["Bing:ApiKey"]; | ||
Assert.NotNull(bingApiKeyCandidate); | ||
this._bingApiKey = bingApiKeyCandidate; | ||
} | ||
|
||
[Theory(Skip = "Requires model deployment that supports function calling.")] | ||
[InlineData("What is the tallest mountain on Earth? How tall is it?", "Everest", new string[] { "WebSearch_Search" })] | ||
[InlineData("What is the weather in Seattle?", "Seattle", new string[] { "WebSearch_Search" })] | ||
[InlineData("What is the current hour number, plus 5?", "", new string[] { "Time_HourNumber", "Math_Add" })] | ||
[InlineData("What is 387 minus 22? Email the solution to John and Mary.", "365", new string[] { "Math_Subtract", "Email_GetEmailAddress", "Email_SendEmail" })] | ||
public async Task CanExecuteStepwisePlanAsync(string prompt, string partialExpectedAnswer, string[] expectedFunctions) | ||
{ | ||
// Arrange | ||
Kernel kernel = this.InitializeKernel(); | ||
var bingConnector = new BingConnector(this._bingApiKey); | ||
var webSearchEnginePlugin = new WebSearchEnginePlugin(bingConnector); | ||
kernel.ImportPluginFromObject(webSearchEnginePlugin, "WebSearch"); | ||
kernel.ImportPluginFromType<TimePlugin>("Time"); | ||
kernel.ImportPluginFromType<MathPlugin>("Math"); | ||
kernel.ImportPluginFromType<EmailPluginFake>("Email"); | ||
|
||
var planner = new FunctionCallingStepwisePlanner( | ||
new FunctionCallingStepwisePlannerConfig() { MaxIterations = 10 }); | ||
|
||
// Act | ||
var planResult = await planner.ExecuteAsync(kernel, prompt); | ||
|
||
// Assert - should contain the expected answer & function calls within the maximum iterations | ||
Assert.NotNull(planResult); | ||
Assert.NotEqual(string.Empty, planResult.FinalAnswer); | ||
Assert.True(planResult.Iterations > 0); | ||
Assert.True(planResult.Iterations <= 10); | ||
Assert.Contains(partialExpectedAnswer, planResult.FinalAnswer, StringComparison.InvariantCultureIgnoreCase); | ||
|
||
string serializedChatHistory = JsonSerializer.Serialize(planResult.ChatHistory); | ||
foreach (string expectedFunction in expectedFunctions) | ||
{ | ||
Assert.Contains(expectedFunction, serializedChatHistory, StringComparison.InvariantCultureIgnoreCase); | ||
} | ||
} | ||
|
||
private Kernel InitializeKernel() | ||
{ | ||
AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get<AzureOpenAIConfiguration>(); | ||
Assert.NotNull(azureOpenAIConfiguration); | ||
|
||
IKernelBuilder builder = Kernel.CreateBuilder() | ||
.AddAzureOpenAIChatCompletion( | ||
deploymentName: azureOpenAIConfiguration.ChatDeploymentName!, | ||
endpoint: azureOpenAIConfiguration.Endpoint, | ||
apiKey: azureOpenAIConfiguration.ApiKey); | ||
|
||
var kernel = builder.Build(); | ||
|
||
return kernel; | ||
} | ||
|
||
private readonly RedirectOutput _testOutputHelper; | ||
private readonly IConfigurationRoot _configuration; | ||
|
||
public void Dispose() | ||
{ | ||
this.Dispose(true); | ||
GC.SuppressFinalize(this); | ||
} | ||
|
||
~FunctionCallingStepwisePlannerTests() | ||
{ | ||
this.Dispose(false); | ||
} | ||
|
||
private void Dispose(bool disposing) | ||
{ | ||
if (disposing) | ||
{ | ||
this._testOutputHelper.Dispose(); | ||
} | ||
} | ||
} |