CopilotChat: External skills and planner

samples/apps/copilot-chat-app/webapi/Config/SequentialPlannerOptions.cs

@@ -0,0 +1,88 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.ComponentModel.DataAnnotations;
+using Microsoft.SemanticKernel.Planning.Sequential;
+
+namespace SemanticKernel.Service.Config;
+
+/// <summary>
+/// Configuration options for the planner.
+/// </summary>
+public class SequentialPlannerOptions
+{
+    public const string PropertyName = "Planner";
+
+    /// <summary>
+    /// Whether to enable the planner.
+    /// </summary>
+    public bool Enabled { get; set; } = false;
+
+    /// <summary>
+    /// The directory containing semantic skills to include in the planner's list of available functions.
+    /// </summary>
+    public string? SemanticSkillsDirectory { get; set; }
+
+    /// <summary>
+    /// The minimum relevancy score for a function to be considered
+    /// </summary>
+    public double? RelevancyThreshold { get; set; }
+
+    /// <summary>
+    /// The maximum number of relevant functions to include in the plan.
+    /// </summary>
+    public int MaxRelevantFunctions { get; set; } = 100;
+
+    /// <summary>
+    /// A list of skills to exclude from the plan creation request.
+    /// </summary>
+    public HashSet<string> ExcludedSkills { get; set; } = new() { };
+
+    /// <summary>
+    /// A list of functions to exclude from the plan creation request.
+    /// </summary>
+    public HashSet<string> ExcludedFunctions { get; set; } = new() { };
+
+    /// <summary>
+    /// A list of functions to include in the plan creation request.
+    /// </summary>
+    public HashSet<string> IncludedFunctions { get; set; } = new() { };
+
+    /// <summary>
+    /// The maximum number of tokens to allow in a plan.
+    /// </summary>
+    [Range(1, int.MaxValue)]
+    public int MaxTokens { get; set; } = 1024;
+
+    /// <summary>
+    /// Convert to a <see cref="SequentialPlannerConfig"/> instance.
+    /// </summary>
+    public SequentialPlannerConfig ToSequentialPlannerConfig()
+    {
+        SequentialPlannerConfig config = new()
+        {
+            RelevancyThreshold = RelevancyThreshold,
+            MaxRelevantFunctions = MaxRelevantFunctions,
+            MaxTokens = MaxTokens,
+        };
+
+        this.ExcludedSkills.Clear();
+        foreach (var excludedSkill in this.ExcludedSkills)
+        {
+            config.ExcludedSkills.Add(excludedSkill);
+        }
+
+        this.ExcludedFunctions.Clear();
+        foreach (var excludedFunction in this.ExcludedFunctions)
+        {
+            config.ExcludedFunctions.Add(excludedFunction);
+        }
+
+        this.IncludedFunctions.Clear();
+        foreach (var includedFunction in this.IncludedFunctions)
+        {
+            config.IncludedFunctions.Add(includedFunction);
+        }
+
+        return config;
+    }
+}

samples/apps/copilot-chat-app/webapi/Controllers/SemanticKernelController.cs

@@ -44,6 +44,8 @@ public class SemanticKernelController : ControllerBase
     /// <param name="chatRepository">Storage repository to store chat sessions</param>
     /// <param name="chatMessageRepository">Storage repository to store chat messages</param>
     /// <param name="documentMemoryOptions">Options for document memory handling.</param>
+    /// <param name="plannerFactory">Factory for planners to use to create function sequences.</param>
+    /// <param name="plannerOptions">Options for the planner.</param>
     /// <param name="ask">Prompt along with its parameters</param>
     /// <param name="skillName">Skill in which function to invoke resides</param>
     /// <param name="functionName">Name of function to invoke</param>
@@ -59,6 +61,8 @@ public class SemanticKernelController : ControllerBase
         [FromServices] ChatSessionRepository chatRepository,
         [FromServices] ChatMessageRepository chatMessageRepository,
         [FromServices] IOptions<DocumentMemoryOptions> documentMemoryOptions,
+        [FromServices] PlannerFactoryAsync plannerFactory,
+        [FromServices] IOptions<SequentialPlannerOptions> plannerOptions,
         [FromBody] Ask ask,
         string skillName, string functionName)
     {
@@ -69,12 +73,20 @@ public class SemanticKernelController : ControllerBase
             return this.BadRequest("Input is required.");
         }
 
+        // Not required for Copilot Chat, but this is how to register additional skills for the service to provide.
         if (!string.IsNullOrWhiteSpace(this._options.SemanticSkillsDirectory))
         {
             kernel.RegisterSemanticSkills(this._options.SemanticSkillsDirectory, this._logger);
         }
 
-        kernel.RegisterNativeSkills(chatRepository, chatMessageRepository, this._promptSettings, documentMemoryOptions.Value, this._logger);
+        kernel.RegisterNativeSkills(
+            chatSessionRepository: chatRepository,
+            chatMessageRepository: chatMessageRepository,
+            promptSettings: this._promptSettings,
+            plannerFactory: plannerFactory,
+            plannerOptions: plannerOptions.Value,
+            documentMemoryOptions: documentMemoryOptions.Value,
+            logger: this._logger);
 
         ISKFunction? function = null;
         try

samples/apps/copilot-chat-app/webapi/CopilotChatApi.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk.Web">
+<Project Sdk="Microsoft.NET.Sdk.Web">
   <PropertyGroup>
     <RepoRoot>$([System.IO.Path]::GetDirectoryName($([MSBuild]::GetPathOfFileAbove('.gitignore', '$(MSBuildThisFileDirectory)'))))</RepoRoot>
   </PropertyGroup>
@@ -13,22 +13,40 @@
     <UserSecretsId>aspnet-SKWebApi-1581687a-bee4-40ea-a886-ce22524aea88</UserSecretsId>
   </PropertyGroup>
 
+  <ItemGroup>
+    <None Include="PlannerSkills\Summarize\config.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="PlannerSkills\Summarize\skprompt.txt">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Include="PlannerSkills\Translate\config.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+    <None Update="PlannerSkills\Translate\skprompt.txt">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
   <ItemGroup>
     <PackageReference Include="Azure.Extensions.AspNetCore.Configuration.Secrets" Version="1.2.2" />
+    <PackageReference Include="Azure.Identity" Version="1.8.2" />
     <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.14" />
     <PackageReference Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" Version="6.0.14" />
     <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.32.3" />
     <PackageReference Include="Microsoft.CognitiveServices.Speech" Version="1.27.0" />
     <PackageReference Include="Microsoft.Extensions.Options.DataAnnotations" Version="7.0.0" />
     <PackageReference Include="Microsoft.Identity.Web" Version="2.9.0" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.5.0" />
-    <PackageReference Include="System.Linq.Async" Version="6.0.1" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\..\..\..\dotnet\src\Connectors\Connectors.AI.OpenAI\Connectors.AI.OpenAI.csproj" />
     <ProjectReference Include="..\..\..\..\dotnet\src\Connectors\Connectors.Memory.Qdrant\Connectors.Memory.Qdrant.csproj" />
+    <ProjectReference Include="..\..\..\..\dotnet\src\Extensions\Planning.SequentialPlanner\Planning.SequentialPlanner.csproj" />
     <ProjectReference Include="..\..\..\..\dotnet\src\SemanticKernel\SemanticKernel.csproj" />
+    <ProjectReference Include="..\..\..\..\dotnet\src\Skills\Skills.OpenAPI\Skills.OpenAPI.csproj" />
+    <ProjectReference Include="..\..\..\..\dotnet\src\Skills\Skills.Web\Skills.Web.csproj" />
   </ItemGroup>
 
 </Project>

samples/apps/copilot-chat-app/webapi/FunctionLoadingExtensions.cs

@@ -42,6 +42,8 @@ internal static class FunctionLoadingExtensions
         ChatSessionRepository chatSessionRepository,
         ChatMessageRepository chatMessageRepository,
         PromptSettings promptSettings,
+        PlannerFactoryAsync plannerFactory,
+        SequentialPlannerOptions plannerOptions,
         DocumentMemoryOptions documentMemoryOptions,
         ILogger logger)
     {
@@ -51,10 +53,13 @@ internal static class FunctionLoadingExtensions
         kernel.ImportSkill(timeSkill, nameof(TimeSkill));
 
         var chatSkill = new ChatSkill(
-            kernel,
-            chatMessageRepository,
-            chatSessionRepository,
-            promptSettings
+            kernel: kernel,
+            chatMessageRepository: chatMessageRepository,
+            chatSessionRepository: chatSessionRepository,
+            promptSettings: promptSettings,
+            plannerFactory: plannerFactory,
+            plannerOptions: plannerOptions,
+            logger: logger
         );
         kernel.ImportSkill(chatSkill, nameof(ChatSkill));
 

samples/apps/copilot-chat-app/webapi/PlannerSkills/Summarize/config.json

@@ -0,0 +1,21 @@
+{
+  "schema": 1,
+  "type": "completion",
+  "description": "Summarize given text or any text document",
+  "completion": {
+    "max_tokens": 512,
+    "temperature": 0.0,
+    "top_p": 0.0,
+    "presence_penalty": 0.0,
+    "frequency_penalty": 0.0
+  },
+  "input": {
+    "parameters": [
+      {
+        "name": "input",
+        "description": "Text to summarize",
+        "defaultValue": ""
+      }
+    ]
+  }
+}

samples/apps/copilot-chat-app/webapi/PlannerSkills/Summarize/skprompt.txt

@@ -0,0 +1,23 @@
+[SUMMARIZATION RULES]
+DONT WASTE WORDS
+USE SHORT, CLEAR, COMPLETE SENTENCES.
+DO NOT USE BULLET POINTS OR DASHES.
+USE ACTIVE VOICE.
+MAXIMIZE DETAIL, MEANING.
+FOCUS ON THE CONTENT
+
+[BANNED PHRASES]
+This article
+This document
+This page
+This material
+[END LIST]
+
+Summarize:
+Hello how are you?
++++++
+Hello
+
+Summarize this
+{{$input}}
++++++

samples/apps/copilot-chat-app/webapi/PlannerSkills/Translate/config.json

@@ -0,0 +1,15 @@
+{
+  "schema": 1,
+  "type": "completion",
+  "description": "Translate the input into a language of your choice",
+  "completion": {
+    "max_tokens": 256,
+    "temperature": 0.7,
+    "top_p": 0.0,
+    "presence_penalty": 0.0,
+    "frequency_penalty": 0.0,
+    "stop_sequences": [
+      "[done]"
+    ]
+  }
+}

samples/apps/copilot-chat-app/webapi/PlannerSkills/Translate/skprompt.txt

@@ -0,0 +1,7 @@
+Translate the input below into {{$language}}
+
+MAKE SURE YOU ONLY USE {{$language}}.
+
+{{$input}}
+
+Translation:

samples/apps/copilot-chat-app/webapi/SemanticKernelExtensions.cs

@@ -8,6 +8,8 @@
 using Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextEmbedding;
 using Microsoft.SemanticKernel.Connectors.Memory.Qdrant;
 using Microsoft.SemanticKernel.Memory;
+using Microsoft.SemanticKernel.Planning;
+using Microsoft.SemanticKernel.Planning.Sequential;
 using Microsoft.SemanticKernel.Reliability;
 using Microsoft.SemanticKernel.SkillDefinition;
 using Microsoft.SemanticKernel.TemplateEngine;
@@ -34,6 +36,7 @@ internal static IServiceCollection AddSemanticKernelServices(this IServiceCollec
         });
         services.AddSingleton<PromptSettings>();
 
+        // Add the semantic memory with backing memory store.
         services.AddSingleton<IMemoryStore>(serviceProvider =>
         {
             MemoriesStoreOptions config = serviceProvider.GetRequiredService<IOptions<MemoriesStoreOptions>>().Value;
@@ -62,9 +65,12 @@ internal static IServiceCollection AddSemanticKernelServices(this IServiceCollec
         });
 
         services.AddScoped<ISemanticTextMemory>(serviceProvider => new SemanticTextMemory(
-            serviceProvider.GetRequiredService<IMemoryStore>(),
-            serviceProvider.GetRequiredService<IOptionsSnapshot<AIServiceOptions>>().Get(AIServiceOptions.EmbeddingPropertyName)
-                .ToTextEmbeddingsService(serviceProvider.GetRequiredService<ILogger<AIServiceOptions>>())));
+                serviceProvider.GetRequiredService<IMemoryStore>(),
+                serviceProvider.GetRequiredService<IOptionsSnapshot<AIServiceOptions>>().Get(AIServiceOptions.EmbeddingPropertyName)
+                    .ToTextEmbeddingsService(serviceProvider.GetRequiredService<ILogger<AIServiceOptions>>())));
+
+        // Add the planner factory.
+        services.AddPlannerFactory();
 
         // Add the Semantic Kernel
         services.AddSingleton<IPromptTemplateEngine, PromptTemplateEngine>();
@@ -77,6 +83,39 @@ internal static IServiceCollection AddSemanticKernelServices(this IServiceCollec
         return services;
     }
 
+    /// <summary>
+    /// Add the planner factory.
+    /// </summary>
+    internal static IServiceCollection AddPlannerFactory(this IServiceCollection services)
+    {
+        // TODO Replace sequential planner with a custom CopilotChat planner tuned to chat scenarios.
+
+        services.AddSingleton<SequentialPlannerConfig>(sp => sp.GetRequiredService<IOptions<SequentialPlannerOptions>>().Value.ToSequentialPlannerConfig());
+        services.AddScoped<PlannerFactoryAsync>(sp => async (IKernel kernel) =>
+        {
+            // Create a kernel for the planner with the same contexts as the chat's kernel but with only skills we want available to the planner.
+            IKernel plannerKernel = new Kernel(new SkillCollection(), kernel.PromptTemplateEngine, kernel.Memory, kernel.Config, kernel.Log);
+
+            //
+            // Add skills to the planner here.
+            //
+            await plannerKernel.ImportChatGptPluginSkillFromUrlAsync("Klarna", new Uri("https://www.klarna.com/.well-known/ai-plugin.json")); // Klarna
+            plannerKernel.ImportSkill(new Microsoft.SemanticKernel.CoreSkills.TextSkill(), "text");
+            plannerKernel.ImportSkill(new Microsoft.SemanticKernel.CoreSkills.TimeSkill(), "time");
+            plannerKernel.ImportSkill(new Microsoft.SemanticKernel.CoreSkills.MathSkill(), "math");
+
+            SequentialPlannerOptions plannerOptions = sp.GetRequiredService<IOptions<SequentialPlannerOptions>>().Value;
+            if (!string.IsNullOrWhiteSpace(plannerOptions.SemanticSkillsDirectory))
+            {
+                plannerKernel.RegisterSemanticSkills(plannerOptions.SemanticSkillsDirectory, sp.GetRequiredService<ILogger>());
+            }
+
+            return new SequentialPlanner(plannerKernel, plannerOptions.ToSequentialPlannerConfig());
+        });
+
+        return services;
+    }
+
     /// <summary>
     /// Add the completion backend to the kernel config
     /// </summary>

samples/apps/copilot-chat-app/webapi/ServiceExtensions.cs

@@ -47,15 +47,21 @@ internal static IServiceCollection AddOptions(this IServiceCollection services,
             .Bind(configuration.GetSection(AzureSpeechOptions.PropertyName))
             .ValidateDataAnnotations().ValidateOnStart();
 
-        // Azure speech token configuration
+        // Bot schema configuration
         services.AddOptions<BotSchemaOptions>()
             .Bind(configuration.GetSection(BotSchemaOptions.PropertyName))
             .ValidateDataAnnotations().ValidateOnStart();
 
+        // Document memory options
         services.AddOptions<DocumentMemoryOptions>()
             .Bind(configuration.GetSection(DocumentMemoryOptions.PropertyName))
             .ValidateDataAnnotations().ValidateOnStart();
 
+        // Planner options
+        services.AddOptions<SequentialPlannerOptions>()
+            .Bind(configuration.GetSection(SequentialPlannerOptions.PropertyName))
+            .ValidateDataAnnotations().ValidateOnStart();
+
         return services;
     }