.Net: Handlebars Prompt Template Helpers (#4017)

### Motivation and Context

<!-- Thank you for your contribution to the semantic-kernel repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->

This PR extends the Handlebars Template Factory with built-in and custom
helpers to provide a consistent and convenient way for prompt and
planner engineers to write Handlebars prompt templates.

These helpers bake in functionality for:
- Marking a block of text as a message with a role for chat completion
connectors.
- Invoking functions from the kernel and passing parameters to them.
- Setting and getting variables in the template context.
- Performing common operations such as concatenation, arithmetic,
comparison, and JSON serialization.
- Supporting different output types and formats for the rendered
template.

ADR here: https://github.com/microsoft/semantic-kernel/pull/3600/files

### Description

<!-- Describe your changes, the overall approach, the underlying design.
These notes will help understanding how your code works. Thanks! -->

Effectively, there are four buckets of helpers enabled in the Handlebars
Template Engine:

1. Default helpers from the Handlebars library, including:
- [Built-in
helpers](https://handlebarsjs.com/guide/builtin-helpers.html) that
enable loops and conditions (#if, #each, #with, #unless)
-
[Handlebars.Net.Helpers](https://github.com/Handlebars-Net/Handlebars.Net.Helpers/wiki)
2. Functions in the kernel
3. Helpers helpful to prompt engineers (i.e., message, or)
4. Utility helpers that can be used to perform simple logic or
transformations on the template data or arguments (i.e., set, get, json,
concat, equals, range, array)

When importing kernel functions as helpers, parameters are checked both
positionally and as hash arguments to ensure expected types match and
all required parameters are present.

This PR also refactors the Handlebars Planner to leverage the new
Handlebars Prompt Template Factory.

Some follow-up (marked with TODOs)
- Isolate Handlebars Kernel System helpers in their own class (Issue
#3947)
- Support override of default options
- Add Categories filter for KernelSystemHelpers (i.e.,
KernelHelperCategories)
- Add support for complex types (in parameter type checks)

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [x] The code builds clean without any errors or warnings
- [x] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [x] All unit tests pass, and I have added new tests where possible
- [x] I didn't break anyone 😄

---------

Co-authored-by: Mark Wallace <127216156+markwallace-microsoft@users.noreply.github.com>

dotnet/Directory.Packages.props

@@ -9,6 +9,7 @@
     <PackageVersion Include="Azure.Identity" Version="1.10.4" />
     <PackageVersion Include="Azure.Monitor.OpenTelemetry.Exporter" Version="1.1.0" />
     <PackageVersion Include="Azure.Search.Documents" Version="11.5.0-beta.5" />
+    <PackageVersion Include="Handlebars.Net.Helpers" Version="2.4.1" />
     <PackageVersion Include="Markdig" Version="0.33.0" />
     <PackageVersion Include="Handlebars.Net" Version="2.1.4" />
     <PackageVersion Include="JsonSchema.Net.Generation" Version="3.5.1" />

dotnet/SK-dotnet.sln

@@ -150,7 +150,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Experimental.Assistants.Uni
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Memory.MongoDB", "src\Connectors\Connectors.Memory.MongoDB\Connectors.Memory.MongoDB.csproj", "{6009CC87-32F1-4282-88BB-8E5A7BA12925}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PromptTemplate.Handlebars", "src\Extensions\PromptTemplate.Handlebars\PromptTemplate.Handlebars.csproj", "{B0646036-0C50-4F66-B479-ADA9C1166816}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PromptTemplates.Handlebars", "src\Extensions\PromptTemplates.Handlebars\PromptTemplates.Handlebars.csproj", "{B0646036-0C50-4F66-B479-ADA9C1166816}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Functions.Yaml", "src\Functions\Functions.Yaml\Functions.Yaml.csproj", "{4AD4E731-16E7-4A0E-B403-6C96459F989B}"
 EndProject

dotnet/samples/KernelSyntaxExamples/Example64_MultiplePromptTemplates.cs

@@ -3,7 +3,7 @@
 using System;
 using System.Threading.Tasks;
 using Microsoft.SemanticKernel;
-using Microsoft.SemanticKernel.PromptTemplate.Handlebars;
+using Microsoft.SemanticKernel.PromptTemplates.Handlebars;
 
 /**
  * This example shows how to use multiple prompt template formats.

dotnet/samples/KernelSyntaxExamples/Example65_HandlebarsPlanner.cs

@@ -25,16 +25,15 @@ public static class Example65_HandlebarsPlanner
     public static async Task RunAsync()
     {
         s_sampleIndex = 1;
-        bool shouldPrintPrompt = true;
+
+        // Using Complex Types as inputs and outputs
+        await RunLocalDictionaryWithComplexTypesSampleAsync(shouldPrintPrompt: true);
 
         // Using primitive types as inputs and outputs
         await PlanNotPossibleSampleAsync();
         await RunDictionaryWithBasicTypesSampleAsync();
         await RunPoetrySampleAsync();
         await RunBookSampleAsync();
-
-        // Using Complex Types as inputs and outputs
-        await RunLocalDictionaryWithComplexTypesSampleAsync(shouldPrintPrompt);
     }
 
     private static void WriteSampleHeadingToConsole(string name)
@@ -93,7 +92,7 @@ await kernel.ImportPluginFromOpenApiAsync(
         // Older models like gpt-35-turbo are less recommended. They do handle loops but are more prone to syntax errors.
         var allowLoopsInPlan = chatDeploymentName.Contains("gpt-4", StringComparison.OrdinalIgnoreCase);
         var planner = new HandlebarsPlanner(
-            new HandlebarsPlannerConfig()
+            new HandlebarsPlannerOptions()
             {
                 // Change this if you want to test with loops regardless of model selection.
                 AllowLoops = allowLoopsInPlan
@@ -105,7 +104,7 @@ await kernel.ImportPluginFromOpenApiAsync(
         var plan = await planner.CreatePlanAsync(kernel, goal);
 
         // Print the prompt template
-        if (shouldPrintPrompt)
+        if (shouldPrintPrompt && plan.Prompt is not null)
         {
             Console.WriteLine($"\nPrompt template:\n{plan.Prompt}");
         }
@@ -126,7 +125,9 @@ private static async Task PlanNotPossibleSampleAsync(bool shouldPrintPrompt = fa
             // Load additional plugins to enable planner but not enough for the given goal.
             await RunSampleAsync("Send Mary an email with the list of meetings I have scheduled today.", shouldPrintPrompt, "SummarizePlugin");
         }
-        catch (KernelException e)
+        catch (KernelException ex) when (
+            ex.Message.Contains(nameof(HandlebarsPlannerErrorCodes.InsufficientFunctionsForGoal), StringComparison.CurrentCultureIgnoreCase)
+            || ex.Message.Contains(nameof(HandlebarsPlannerErrorCodes.HallucinatedHelpers), StringComparison.CurrentCultureIgnoreCase))
         {
             /*
                 Unable to create plan for goal with available functions.
@@ -137,7 +138,7 @@ private static async Task PlanNotPossibleSampleAsync(bool shouldPrintPrompt = fa
                 Therefore, I cannot create a Handlebars template to achieve the specified goal with the available helpers. 
                 Additional helpers may be required.
             */
-            Console.WriteLine($"{e.Message}\n");
+            Console.WriteLine($"\n{ex.Message}\n");
         }
     }
 

dotnet/samples/KernelSyntaxExamples/Getting_Started/Step3_Yaml_Prompt.cs

@@ -3,7 +3,7 @@
 using System;
 using System.Threading.Tasks;
 using Microsoft.SemanticKernel;
-using Microsoft.SemanticKernel.PromptTemplate.Handlebars;
+using Microsoft.SemanticKernel.PromptTemplates.Handlebars;
 
 /**
  * This example shows how to create a prompt <see cref="KernelFunction"/> from a YAML resource.

dotnet/samples/KernelSyntaxExamples/KernelSyntaxExamples.csproj

@@ -54,7 +54,7 @@
     <ProjectReference Include="..\..\src\Connectors\Connectors.Memory.Redis\Connectors.Memory.Redis.csproj" />
     <ProjectReference Include="..\..\src\Connectors\Connectors.Memory.Pinecone\Connectors.Memory.Pinecone.csproj" />
     <ProjectReference Include="..\..\src\Experimental\Assistants\Experimental.Assistants.csproj" />
-    <ProjectReference Include="..\..\src\Extensions\PromptTemplate.Handlebars\PromptTemplate.Handlebars.csproj" />
+    <ProjectReference Include="..\..\src\Extensions\PromptTemplates.Handlebars\PromptTemplates.Handlebars.csproj" />
     <ProjectReference Include="..\..\src\Planners\Planners.Handlebars\Planners.Handlebars.csproj" />
     <ProjectReference Include="..\..\src\Planners\Planners.OpenAI\Planners.OpenAI.csproj" />
     <ProjectReference Include="..\..\src\SemanticKernel.Abstractions\SemanticKernel.Abstractions.csproj" />

dotnet/samples/TelemetryExample/Program.cs

@@ -81,7 +81,7 @@ public static async Task Main()
         Console.WriteLine("Original plan:");
         Console.WriteLine(plan.ToString());
 
-        var result = await plan.InvokeAsync(kernel, new KernelArguments(), CancellationToken.None);
+        var result = await plan.InvokeAsync(kernel, new KernelArguments(), CancellationToken.None).ConfigureAwait(false);
 
         Console.WriteLine("Result:");
         Console.WriteLine(result);
@@ -108,7 +108,7 @@ private static Kernel GetKernel(ILoggerFactory loggerFactory)
 
     private static HandlebarsPlanner CreatePlanner(int maxTokens = 1024)
     {
-        var plannerConfig = new HandlebarsPlannerConfig { MaxTokens = maxTokens };
+        var plannerConfig = new HandlebarsPlannerOptions { MaxTokens = maxTokens };
         return new HandlebarsPlanner(plannerConfig);
     }
 

dotnet/src/Extensions/Extensions.UnitTests/Extensions.UnitTests.csproj

@@ -25,6 +25,6 @@
     <PackageReference Include="Microsoft.Extensions.Logging.Console" />
   </ItemGroup>
   <ItemGroup>
-    <ProjectReference Include="..\PromptTemplate.Handlebars\PromptTemplate.Handlebars.csproj" />
+    <ProjectReference Include="..\PromptTemplates.Handlebars\PromptTemplates.Handlebars.csproj" />
   </ItemGroup>
 </Project>

dotnet/src/Extensions/Extensions.UnitTests/PromptTemplate/Handlebars/HandlebarsPromptTemplateFactoryTests.cs → dotnet/src/Extensions/Extensions.UnitTests/PromptTemplates/Handlebars/HandlebarsPromptTemplateFactoryTests.cs

@@ -1,10 +1,12 @@
 // Copyright (c) Microsoft. All rights reserved.
 
 using Microsoft.SemanticKernel;
-using Microsoft.SemanticKernel.PromptTemplate.Handlebars;
+using Microsoft.SemanticKernel.PromptTemplates.Handlebars;
 using Xunit;
 
-namespace SemanticKernel.Extensions.UnitTests.PromptTemplate.Handlebars;
+using static Extensions.UnitTests.PromptTemplates.Handlebars.TestUtilities;
+
+namespace SemanticKernel.Extensions.UnitTests.PromptTemplates.Handlebars;
 
 public sealed class HandlebarsPromptTemplateFactoryTests
 {
@@ -13,7 +15,7 @@ public void ItCreatesHandlebarsPromptTemplate()
     {
         // Arrange
         var templateString = "{{input}}";
-        var promptConfig = new PromptTemplateConfig() { TemplateFormat = HandlebarsPromptTemplateFactory.HandlebarsTemplateFormat, Template = templateString };
+        var promptConfig = InitializeHbPromptConfig(templateString);
         var target = new HandlebarsPromptTemplateFactory();
 
         // Act

dotnet/src/Extensions/Extensions.UnitTests/PromptTemplates/Handlebars/HandlebarsPromptTemplateTestUtils.cs

@@ -0,0 +1,18 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.PromptTemplates.Handlebars;
+
+namespace Extensions.UnitTests.PromptTemplates.Handlebars;
+
+internal static class TestUtilities
+{
+    public static PromptTemplateConfig InitializeHbPromptConfig(string template)
+    {
+        return new PromptTemplateConfig()
+        {
+            TemplateFormat = HandlebarsPromptTemplateFactory.HandlebarsTemplateFormat,
+            Template = template
+        };
+    }
+}