Skip to content

Commit

Permalink
.Net: Integration Tests for Function Calling Stepwise Planner (#4189)
Browse files Browse the repository at this point in the history
### Motivation and Context / Description

<!-- Thank you for your contribution to the semantic-kernel repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->
- The integration tests for Function Calling Stepwise Planner were
removed when we refactored/removed `Planners.Core`. This adds them back
and improves them with more test cases and results validation.
- Also includes a minor fix in sample code (`ChatHistory.AsJson()` no
longer exists)

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [x] The code builds clean without any errors or warnings
- [x] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [x] All unit tests pass, and I have added new tests where possible
- [x] I didn't break anyone 😄
  • Loading branch information
gitri-ms committed Dec 12, 2023
1 parent 13a4b4e commit 38ac687
Show file tree
Hide file tree
Showing 3 changed files with 114 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ public static async Task RunAsync()
Console.WriteLine($"Q: {question}\nA: {result.FinalAnswer}");

// You can uncomment the line below to see the planner's process for completing the request.
// Console.WriteLine($"Chat history:\n{result.ChatHistory?.AsJson()}");
// Console.WriteLine($"Chat history:\n{System.Text.Json.JsonSerializer.Serialize(result.ChatHistory)}");
}
}

Expand Down
2 changes: 1 addition & 1 deletion dotnet/src/IntegrationTests/IntegrationTests.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
<RollForward>LatestMajor</RollForward>
<IsTestProject>true</IsTestProject>
<IsPackable>false</IsPackable>
<NoWarn>CA2007,VSTHRD111,SKEXP0001,SKEXP0002,SKEXP0003,SKEXP0004,SKEXP0010,SKEXP0011,SKEXP0012,SKEXP0020,SKEXP0021,SKEXP0022,SKEXP0023,SKEXP0024,SKEXP0025,SKEXP0026,SKEXP0027,SKEXP0028,SKEXP0029,SKEXP0030,SKEXP0031,SKEXP0032,SKEXP0060</NoWarn>
<NoWarn>CA2007,VSTHRD111,SKEXP0001,SKEXP0002,SKEXP0003,SKEXP0004,SKEXP0010,SKEXP0011,SKEXP0012,SKEXP0020,SKEXP0021,SKEXP0022,SKEXP0023,SKEXP0024,SKEXP0025,SKEXP0026,SKEXP0027,SKEXP0028,SKEXP0029,SKEXP0030,SKEXP0031,SKEXP0032,SKEXP0050,SKEXP0054,SKEXP0060,SKEXP0061</NoWarn>
<UserSecretsId>b7762d10-e29b-4bb1-8b74-b6d69a667dd4</UserSecretsId>
</PropertyGroup>
<ItemGroup>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
// Copyright (c) Microsoft. All rights reserved.

using System;
using System.Threading.Tasks;
using Microsoft.Extensions.Configuration;
using Microsoft.SemanticKernel.Planning;
using Microsoft.SemanticKernel.Plugins.Core;
using Microsoft.SemanticKernel.Plugins.Web.Bing;
using Microsoft.SemanticKernel.Plugins.Web;
using Microsoft.SemanticKernel;
using SemanticKernel.IntegrationTests.TestSettings;
using Xunit.Abstractions;
using Xunit;
using SemanticKernel.IntegrationTests.Fakes;
using System.Text.Json;

namespace SemanticKernel.IntegrationTests.Planners.Stepwise;
public sealed class FunctionCallingStepwisePlannerTests : IDisposable
{
private readonly string _bingApiKey;

public FunctionCallingStepwisePlannerTests(ITestOutputHelper output)
{
this._testOutputHelper = new RedirectOutput(output);

// Load configuration
this._configuration = new ConfigurationBuilder()
.AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true)
.AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true)
.AddEnvironmentVariables()
.AddUserSecrets<FunctionCallingStepwisePlannerTests>()
.Build();

string? bingApiKeyCandidate = this._configuration["Bing:ApiKey"];
Assert.NotNull(bingApiKeyCandidate);
this._bingApiKey = bingApiKeyCandidate;
}

[Theory(Skip = "Requires model deployment that supports function calling.")]
[InlineData("What is the tallest mountain on Earth? How tall is it?", "Everest", new string[] { "WebSearch_Search" })]
[InlineData("What is the weather in Seattle?", "Seattle", new string[] { "WebSearch_Search" })]
[InlineData("What is the current hour number, plus 5?", "", new string[] { "Time_HourNumber", "Math_Add" })]
[InlineData("What is 387 minus 22? Email the solution to John and Mary.", "365", new string[] { "Math_Subtract", "Email_GetEmailAddress", "Email_SendEmail" })]
public async Task CanExecuteStepwisePlanAsync(string prompt, string partialExpectedAnswer, string[] expectedFunctions)
{
// Arrange
Kernel kernel = this.InitializeKernel();
var bingConnector = new BingConnector(this._bingApiKey);
var webSearchEnginePlugin = new WebSearchEnginePlugin(bingConnector);
kernel.ImportPluginFromObject(webSearchEnginePlugin, "WebSearch");
kernel.ImportPluginFromType<TimePlugin>("Time");
kernel.ImportPluginFromType<MathPlugin>("Math");
kernel.ImportPluginFromType<EmailPluginFake>("Email");

var planner = new FunctionCallingStepwisePlanner(
new FunctionCallingStepwisePlannerConfig() { MaxIterations = 10 });

// Act
var planResult = await planner.ExecuteAsync(kernel, prompt);

// Assert - should contain the expected answer & function calls within the maximum iterations
Assert.NotNull(planResult);
Assert.NotEqual(string.Empty, planResult.FinalAnswer);
Assert.True(planResult.Iterations > 0);
Assert.True(planResult.Iterations <= 10);
Assert.Contains(partialExpectedAnswer, planResult.FinalAnswer, StringComparison.InvariantCultureIgnoreCase);

string serializedChatHistory = JsonSerializer.Serialize(planResult.ChatHistory);
foreach (string expectedFunction in expectedFunctions)
{
Assert.Contains(expectedFunction, serializedChatHistory, StringComparison.InvariantCultureIgnoreCase);
}
}

private Kernel InitializeKernel()
{
AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get<AzureOpenAIConfiguration>();
Assert.NotNull(azureOpenAIConfiguration);

IKernelBuilder builder = Kernel.CreateBuilder()
.AddAzureOpenAIChatCompletion(
deploymentName: azureOpenAIConfiguration.ChatDeploymentName!,
endpoint: azureOpenAIConfiguration.Endpoint,
apiKey: azureOpenAIConfiguration.ApiKey);

var kernel = builder.Build();

return kernel;
}

private readonly RedirectOutput _testOutputHelper;
private readonly IConfigurationRoot _configuration;

public void Dispose()
{
this.Dispose(true);
GC.SuppressFinalize(this);
}

~FunctionCallingStepwisePlannerTests()
{
this.Dispose(false);
}

private void Dispose(bool disposing)
{
if (disposing)
{
this._testOutputHelper.Dispose();
}
}
}

0 comments on commit 38ac687

Please sign in to comment.