.Net: Function calling stepwise planner integration tests in pipeline

.github/workflows/dotnet-build-and-test.yml

@@ -51,11 +51,7 @@ jobs:
         include:
           - { dotnet: "6.0-jammy", os: "ubuntu", configuration: Debug }
           - { dotnet: "7.0-jammy", os: "ubuntu", configuration: Release }
-          - {
-              dotnet: "8.0-jammy",
-              os: "ubuntu",
-              configuration: Release,
-            }
+          - { dotnet: "8.0-jammy", os: "ubuntu", configuration: Release }
           - { dotnet: "6.0", os: "windows", configuration: Release }
           - {
               dotnet: "7.0",
@@ -105,6 +101,9 @@ jobs:
           AzureOpenAIEmbeddings__Endpoint: ${{ secrets.AZUREOPENAI__ENDPOINT }}
           AzureOpenAI__ApiKey: ${{ secrets.AZUREOPENAI__APIKEY }}
           AzureOpenAIEmbeddings__ApiKey: ${{ secrets.AZUREOPENAI__APIKEY }}
+          Planners__AzureOpenAI__ApiKey: ${{ secrets.PLANNERS__AZUREOPENAI__APIKEY }}
+          Planners__AzureOpenAI__Endpoint: ${{ secrets.PLANNERS__AZUREOPENAI__ENDPOINT }}
+          Planners__AzureOpenAI__DeploymentName: ${{ vars.PLANNERS__AZUREOPENAI__DEPLOYMENTNAME }}
           Bing__ApiKey: ${{ secrets.BING__APIKEY }}
           OpenAI__ApiKey: ${{ secrets.OPENAI__APIKEY }}
 

.github/workflows/dotnet-integration-tests.yml

@@ -21,7 +21,7 @@ jobs:
         os: [ubuntu-latest]
         configuration: [Debug]
     runs-on: ${{ matrix.os }}
-    steps:        
+    steps:
       - uses: actions/checkout@v4
         if: ${{ github.event_name != 'pull_request' }}
         with:

dotnet/src/IntegrationTests/Planners/Stepwise/FunctionCallingStepwisePlannerTests.cs

@@ -36,12 +36,12 @@ public FunctionCallingStepwisePlannerTests(ITestOutputHelper output)
         this._bingApiKey = bingApiKeyCandidate;
     }
 
-    [Theory(Skip = "Requires model deployment that supports function calling.")]
-    [InlineData("What is the tallest mountain on Earth? How tall is it?", "Everest", new string[] { "WebSearch_Search" })]
-    [InlineData("What is the weather in Seattle?", "Seattle", new string[] { "WebSearch_Search" })]
-    [InlineData("What is the current hour number, plus 5?", "", new string[] { "Time_HourNumber", "Math_Add" })]
-    [InlineData("What is 387 minus 22? Email the solution to John and Mary.", "365", new string[] { "Math_Subtract", "Email_GetEmailAddress", "Email_SendEmail" })]
-    public async Task CanExecuteStepwisePlanAsync(string prompt, string partialExpectedAnswer, string[] expectedFunctions)
+    [Theory]
+    [InlineData("What is the tallest mountain on Earth? How tall is it?", new string[] { "WebSearch_Search" })]
+    [InlineData("What is the weather in Seattle?", new string[] { "WebSearch_Search" })]
+    [InlineData("What is the current hour number, plus 5?", new string[] { "Time_HourNumber", "Math_Add" })]
+    [InlineData("What is 387 minus 22? Email the solution to John and Mary.", new string[] { "Math_Subtract", "Email_GetEmailAddress", "Email_SendEmail" })]
+    public async Task CanExecuteStepwisePlanAsync(string prompt, string[] expectedFunctions)
     {
         // Arrange
         Kernel kernel = this.InitializeKernel();
@@ -63,7 +63,7 @@ public async Task CanExecuteStepwisePlanAsync(string prompt, string partialExpec
         Assert.NotEqual(string.Empty, planResult.FinalAnswer);
         Assert.True(planResult.Iterations > 0);
         Assert.True(planResult.Iterations <= 10);
-        Assert.Contains(partialExpectedAnswer, planResult.FinalAnswer, StringComparison.InvariantCultureIgnoreCase);
+        Assert.NotEmpty(planResult.FinalAnswer);
 
         string serializedChatHistory = JsonSerializer.Serialize(planResult.ChatHistory);
         foreach (string expectedFunction in expectedFunctions)
@@ -74,7 +74,7 @@ public async Task CanExecuteStepwisePlanAsync(string prompt, string partialExpec
 
     private Kernel InitializeKernel()
     {
-        AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get<AzureOpenAIConfiguration>();
+        AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("Planners:AzureOpenAI").Get<AzureOpenAIConfiguration>();
         Assert.NotNull(azureOpenAIConfiguration);
 
         IKernelBuilder builder = Kernel.CreateBuilder()

dotnet/src/IntegrationTests/TestSettings/PlannerConfiguration.cs

@@ -0,0 +1,20 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Diagnostics.CodeAnalysis;
+
+namespace SemanticKernel.IntegrationTests.TestSettings;
+
+[SuppressMessage("Performance", "CA1812:Internal class that is apparently never instantiated",
+    Justification = "Configuration classes are instantiated through IConfiguration.")]
+internal sealed class PlannerConfiguration
+{
+    public AzureOpenAIConfiguration? AzureOpenAI { get; set; }
+
+    public OpenAIConfiguration? OpenAI { get; set; }
+
+    public PlannerConfiguration(AzureOpenAIConfiguration? azureOpenAIConfiguration, OpenAIConfiguration? openAIConfiguration)
+    {
+        this.AzureOpenAI = azureOpenAIConfiguration;
+        this.OpenAI = openAIConfiguration;
+    }
+}

dotnet/src/IntegrationTests/testsettings.json

@@ -34,5 +34,18 @@
   "MongoDB": {
     "ConnectionString": "",
     "VectorSearchCollection": "dotnetMSKNearestTest.nearestSearch"
+  },
+  "Planners": {
+    "AzureOpenAI": {
+      "ServiceId": "azure-gpt-35-turbo",
+      "DeploymentName": "gpt-35-turbo",
+      "Endpoint": "",
+      "ApiKey": ""
+    },
+    "OpenAI": {
+      "ServiceId": "text-davinci-003",
+      "ModelId": "text-davinci-003",
+      "ApiKey": ""
+    }
   }
 }

dotnet/src/Planners/Planners.OpenAI/Stepwise/GeneratePlan.yaml

@@ -1,6 +1,6 @@
 template_format: semantic-kernel
 template: |
-  <message role=""system"">
+  <message role="system">
   You are an expert at generating plans from a given GOAL. Think step by step and determine a plan to satisfy the specified GOAL using only the FUNCTIONS provided to you. You can also make use of your own knowledge while forming an answer but you must not use functions that are not provided. Once you have come to a final answer, use the UserInteraction_SendFinalAnswer function to communicate this back to the user.
 
   [FUNCTIONS]
@@ -14,7 +14,7 @@ template: |
   1. Steps can use output from one or more previous steps as input, if appropriate.
   2. The plan should be as short as possible.
   </message>
-  <message role=""user"">{{$goal}}</message>
+  <message role="user">{{$goal}}</message>
 description:     Generate a step-by-step plan to satisfy a given goal
 name:            GeneratePlan
 input_variables:
@@ -23,7 +23,8 @@ input_variables:
   - name:          goal
     description:   The goal to satisfy
 execution_settings:
-  - temperature:       0.0
+  default:  
+    temperature:       0.0
     top_p:             0.0
     presence_penalty:  0.0
     frequency_penalty: 0.0