.Net: Tuning HB Planner - CreatePlan prompt, template extraction, and…

… type handling (#5137)

### Motivation and Context

<!-- Thank you for your contribution to the semantic-kernel repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->
Resolves 
- #4751 
- #4250 
- #4721

This PR contains fixes for template extraction from model results;
handling case where schema and parameter type can co-exist; + prompt
tuning

### Description

<!-- Describe your changes, the overall approach, the underlying design.
These notes will help understanding how your code works. Thanks! -->

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [ ] The code builds clean without any errors or warnings
- [ ] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [ ] All unit tests pass, and I have added new tests where possible
- [ ] I didn't break anyone 😄

dotnet/Directory.Packages.props

@@ -9,7 +9,7 @@
     <PackageVersion Include="Azure.Identity" Version="1.10.4" />
     <PackageVersion Include="Azure.Monitor.OpenTelemetry.Exporter" Version="1.2.0" />
     <PackageVersion Include="Azure.Search.Documents" Version="11.5.1" />
-    <PackageVersion Include="Handlebars.Net.Helpers" Version="2.4.1.3" />
+    <PackageVersion Include="Handlebars.Net.Helpers" Version="2.4.1.4" />
     <PackageVersion Include="Markdig" Version="0.34.0" />
     <PackageVersion Include="Handlebars.Net" Version="2.1.4" />
     <PackageVersion Include="JsonSchema.Net.Generation" Version="3.5.1" />

dotnet/samples/KernelSyntaxExamples/Example65_HandlebarsPlanner.cs

@@ -142,26 +142,16 @@ public async Task PlanNotPossibleSampleAsync(bool shouldPrintPrompt)
     {
         WriteSampleHeading("Plan Not Possible");
 
-        try
-        {
-            // Load additional plugins to enable planner but not enough for the given goal.
-            await RunSampleAsync("Send Mary an email with the list of meetings I have scheduled today.", null, null, shouldPrintPrompt, true, "SummarizePlugin");
-        }
-        catch (KernelException ex) when (
-            ex.Message.Contains(nameof(HandlebarsPlannerErrorCodes.InsufficientFunctionsForGoal), StringComparison.CurrentCultureIgnoreCase)
-            || ex.Message.Contains(nameof(HandlebarsPlannerErrorCodes.HallucinatedHelpers), StringComparison.CurrentCultureIgnoreCase)
-            || ex.Message.Contains(nameof(HandlebarsPlannerErrorCodes.InvalidTemplate), StringComparison.CurrentCultureIgnoreCase))
-        {
-            /*
-                [InsufficientFunctionsForGoal] Unable to create plan for goal with available functions.
-                Goal: Send Mary an email with the list of meetings I have scheduled today.
-                Available Functions: SummarizePlugin-MakeAbstractReadable, SummarizePlugin-Notegen, SummarizePlugin-Summarize, SummarizePlugin-Topics
-                Planner output:
-                As the available helpers do not contain any functionality to send an email or interact with meeting scheduling data, I cannot create a template to achieve the stated goal. 
-                Additional helpers or information may be required.
-            */
-            WriteLine($"\n{ex.Message}\n");
-        }
+        // Load additional plugins to enable planner but not enough for the given goal.
+        await RunSampleAsync("Send Mary an email with the list of meetings I have scheduled today.", null, null, shouldPrintPrompt, true, "SummarizePlugin");
+        /*
+            [InsufficientFunctionsForGoal] Unable to create plan for goal with available functions.
+            Goal: Send Mary an email with the list of meetings I have scheduled today.
+            Available Functions: SummarizePlugin-MakeAbstractReadable, SummarizePlugin-Notegen, SummarizePlugin-Summarize, SummarizePlugin-Topics
+            Planner output:
+            As the available helpers do not contain any functionality to send an email or interact with meeting scheduling data, I cannot create a template to achieve the stated goal. 
+            Additional helpers or information may be required.
+        */
     }
 
     [RetryTheory(typeof(HttpOperationException))]

dotnet/src/Planners/Planners.Handlebars.UnitTests/Handlebars/HandlebarsPlannerTests.cs

@@ -193,6 +193,41 @@ public async Task ItOverridesPromptAsync()
         Assert.DoesNotContain("## Tips and reminders", plan.Prompt, StringComparison.CurrentCulture);
     }
 
+    [Fact]
+    public async Task ItThrowsIfStrictlyOnePlanCantBeIdentifiedAsync()
+    {
+        // Arrange
+        var ResponseWithMultipleHbTemplates =
+    @"```handlebars
+{{!-- Step 1: Call Summarize function --}}  
+{{set ""summary"" (SummarizePlugin-Summarize)}}  
+```
+
+```handlebars
+{{!-- Step 2: Call Translate function with the language set to French --}}  
+{{set ""translatedSummary"" (WriterPlugin-Translate language=""French"" input=(get ""summary""))}}  
+```
+
+```handlebars
+{{!-- Step 3: Call GetEmailAddress function with input set to John Doe --}}  
+{{set ""emailAddress"" (email-GetEmailAddress input=""John Doe"")}}  
+
+{{!-- Step 4: Call SendEmail function with input set to the translated summary and email_address set to the retrieved email address --}}  
+{{email-SendEmail input=(get ""translatedSummary"") email_address=(get ""emailAddress"")}}
+```
+
+```handlebars
+{{!-- Step 4: Call SendEmail function with input set to the translated summary and email_address set to the retrieved email address --}}  
+{{email-SendEmail input=(get ""translatedSummary"") email_address=(get ""emailAddress"")}}
+```";
+        var kernel = this.CreateKernelWithMockCompletionResult(ResponseWithMultipleHbTemplates);
+        var planner = new HandlebarsPlanner();
+
+        // Act & Assert
+        var exception = await Assert.ThrowsAsync<PlanCreationException>(async () => await planner.CreatePlanAsync(kernel, "goal"));
+        Assert.True(exception?.InnerException?.Message?.Contains("Identified multiple Handlebars templates in model response", StringComparison.InvariantCulture));
+    }
+
     private Kernel CreateKernelWithMockCompletionResult(string testPlanString, KernelPluginCollection? plugins = null)
     {
         plugins ??= new KernelPluginCollection();

dotnet/src/Planners/Planners.Handlebars/Handlebars/CreatePlanPromptPartials/Helpers/CustomHelpers/CustomHelpers.handlebars

@@ -12,9 +12,9 @@ Description: {{Description}}
 Inputs:
   {{#each Parameters}}
     - {{Name}}:
-    {{~#if ParameterType}} {{ParameterType.Name}} -
+    {{~#if Schema}} {{getSchemaTypeName this}} -
     {{~else}}
-        {{~#if Schema}} {{getSchemaTypeName this}} -{{/if}}
+        {{~#if ParameterType}} {{ParameterType.Name}} -{{/if}}
     {{~/if}}
     {{~#if Description}} {{Description}}{{/if}}
     {{~#if IsRequired}} (required){{else}} (optional){{/if}}

dotnet/src/Planners/Planners.Handlebars/Handlebars/CreatePlanPromptPartials/Helpers/VariableHelpers.handlebars

@@ -3,6 +3,6 @@
 ## Variable helpers
 If you need to create or retrieve a variable, you can use the following helpers:
 - `\{{set}}` – Creates a variable with the given name and value. It does not print anything to the template, so you must use `\{{json}}` to print the value.
-- `\{{json}}` – Generates and prints a JSON string from the given value.
+- `\{{json}}` – Serializes the given value and prints result as JSON string.
 - `\{{concat}}` – Concatenates the given values into one string.
 {{/inline}}

dotnet/src/Planners/Planners.Handlebars/Handlebars/CreatePlanPromptPartials/TipsAndInstructions.handlebars

@@ -15,7 +15,7 @@ Follow these steps to create one Handlebars template to achieve the goal:
   - Omit this step if no values are needed from the initial context.
 1. Choose the Right Helpers:
   - Use the provided helpers to manipulate the variables you've created. Start with the basic helpers and only use custom helpers if necessary to accomplish the goal.
-  - Always reference a custom helper by its full name.
+  - Be careful with syntax, i.e., Always reference a custom helper by its full name and remember to use a `#` for all block helpers.
 2. Don't Create or Assume Unlisted Helpers:
   - Only use the helpers provided. Any helper not listed is considered hallucinated and must not be used.
   - Do not invent or assume the existence of any functions not explicitly defined above.
@@ -27,7 +27,7 @@ Follow these steps to create one Handlebars template to achieve the goal:
 5. No Nested Helpers:
   - Do not nest helpers or conditionals inside other helpers. This can cause errors in the template.
 6. Output the Result:
-  - Once you have completed the necessary steps to reach the goal, use the `\{{json}}` helper to output the final result.
+  - Once you have completed the necessary steps to reach the goal, use the `\{{json}}` helper and print only your final template.
   - Ensure your template and all steps are enclosed in a ``` handlebars block.
 
 Remember, the objective is not to use all the helpers available, but to use the correct ones to achieve the desired outcome with a clear and concise template.

dotnet/src/Planners/Planners.Handlebars/Handlebars/HandlebarsPlanner.cs

@@ -89,13 +89,21 @@ private async Task<HandlebarsPlan> CreatePlanCoreAsync(Kernel kernel, string goa
             var chatCompletionService = kernel.GetRequiredService<IChatCompletionService>();
             modelResults = await chatCompletionService.GetChatMessageContentAsync(chatMessages, executionSettings: this._options.ExecutionSettings, cancellationToken: cancellationToken).ConfigureAwait(false);
 
-            Match match = Regex.Match(modelResults.Content, @"```\s*(handlebars)?\s*(.*)\s*```", RegexOptions.Singleline);
-            if (!match.Success)
+            // Regex breakdown:
+            // (```\s*handlebars){1}\s*: Opening backticks, starting boundary for HB template
+            // ((([^`]|`(?!``))+): Any non-backtick character or one backtick character not followed by 2 more consecutive backticks
+            // (\s*```){1}: Closing backticks, closing boundary for HB template
+            MatchCollection matches = Regex.Matches(modelResults.Content, @"(```\s*handlebars){1}\s*(([^`]|`(?!``))+)(\s*```){1}", RegexOptions.Multiline);
+            if (matches.Count < 1)
             {
-                throw new KernelException($"[{HandlebarsPlannerErrorCodes.InvalidTemplate}] Could not find the plan in the results.");
+                throw new KernelException($"[{HandlebarsPlannerErrorCodes.InvalidTemplate}] Could not find the plan in the results. Additional helpers or input may be required.\n\nPlanner output:\n{modelResults.Content}");
+            }
+            else if (matches.Count > 1)
+            {
+                throw new KernelException($"[{HandlebarsPlannerErrorCodes.InvalidTemplate}] Identified multiple Handlebars templates in model response. Please try again.\n\nPlanner output:\n{modelResults.Content}");
             }
 
-            var planTemplate = match.Groups[2].Value.Trim();
+            var planTemplate = matches[0].Groups[2].Value.Trim();
             planTemplate = MinifyHandlebarsTemplate(planTemplate);
 
             return new HandlebarsPlan(planTemplate, createPlanPrompt);
@@ -153,7 +161,26 @@ private async Task<HandlebarsPlan> CreatePlanCoreAsync(Kernel kernel, string goa
         HashSet<HandlebarsParameterTypeMetadata> complexParameterTypes,
         Dictionary<string, string> complexParameterSchemas)
     {
-        // TODO (@teresaqhoang): Handle case when schema and ParameterType can exist i.e., when ParameterType = RestApiResponse
+        if (parameter.Schema is not null)
+        {
+            // Class types will have a defined schema, but we want to handle those as built-in complex types below
+            if (parameter.ParameterType is not null && parameter.ParameterType!.IsClass)
+            {
+                parameter = new(parameter) { Schema = null };
+            }
+            else
+            {
+                // Parse the schema to extract any primitive types and set in ParameterType property instead
+                var parsedParameter = parameter.ParseJsonSchema();
+                if (parsedParameter.Schema is not null)
+                {
+                    complexParameterSchemas[parameter.GetSchemaTypeName()] = parameter.Schema.RootElement.ToJsonString();
+                }
+
+                return parsedParameter;
+            }
+        }
+
         if (parameter.ParameterType is not null)
         {
             // Async return type - need to extract the actual return type and override ParameterType property
@@ -165,17 +192,6 @@ private async Task<HandlebarsPlan> CreatePlanCoreAsync(Kernel kernel, string goa
 
             complexParameterTypes.UnionWith(parameter.ParameterType!.ToHandlebarsParameterTypeMetadata());
         }
-        else if (parameter.Schema is not null)
-        {
-            // Parse the schema to extract any primitive types and set in ParameterType property instead
-            var parsedParameter = parameter.ParseJsonSchema();
-            if (parsedParameter.Schema is not null)
-            {
-                complexParameterSchemas[parameter.GetSchemaTypeName()] = parameter.Schema.RootElement.ToJsonString();
-            }
-
-            parameter = parsedParameter;
-        }
 
         return parameter;
     }

dotnet/src/Planners/Planners.Handlebars/Handlebars/HandlebarsPlannerOptions.cs

@@ -28,7 +28,8 @@ public sealed class HandlebarsPlannerOptions : PlannerOptions
     /// Delegate that returns an override for the CreatePlan prompt.
     /// </summary>
     /// <remarks>
-    /// Handler will be used as a callback. The callback should return a valid Handlebars template string. If this is set, the planner will use this prompt instead of the default prompt.
+    /// Handler will be used as a callback. The callback should return a valid Handlebars template string in a ```handlebars codeblock.
+    /// If this is set, the planner will use this prompt instead of the default prompt.
     /// Devs can select any partial defined in Planners.Handlebars.CreatePlanPromptPartials namespace when constructing their own prompt.
     /// No partials are included by default; make sure to select partials such as "{{> UserGoal }}" or {{> AdditionalContext}} if needed.
     /// </remarks>