Refactor ChatSkill to use new Chat Completions prompt format (#337)

### Motivation and Context

<!-- Thank you for your contribution to the chat-copilot repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->
This PR refactors the ChatSkill to use the[ new ChatCompletions prompt
format](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/chatgpt?pivots=programming-language-chat-completions#system-role)
as the meta-prompt template. The new format serves the prompt template
as a series of context messages, where each message can be tagged a
role, to prime the model in a certain way. These roles include: System,
User, and Assistant.

### Description

<!-- Describe your changes, the overall approach, the underlying design.
These notes will help understanding how your code works. Thanks! -->

For ChatSkill, the prompt template for the meta-bot response request
utilizes the message roles in the following way:
- System: for any instructions or additional context provided to the
model, including bot persona, relevant memories, or planner output.
- Assistant: Any bot response message in chat history previously
outputted as chat completion result.
- User: Any user input.

The series of messages between the user and the assistant is included
both as few shot examples and additional context (chat history). The
last user message is always included.

Other changes:
- This PR also adds a custom token calculator, since chat streaming
doesn't return token usage details.
- System continuation preamble is no longer needed with this new format,
since it's been optimized for chat responses.
- Added visible scroll bar to prompt dialog.
- Simplified SystemResponse since this conversational interface is
already optimized for chat.

<img width="576" alt="image"
src="https://github.com/microsoft/chat-copilot/assets/125500434/371b672d-85d3-45e4-8375-c8122c0701ef">
<img width="663" alt="image"
src="https://github.com/microsoft/chat-copilot/assets/125500434/6ec8ffe3-f9c6-4ae0-893b-48668f8ce8e0">


### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [x] The code builds clean without any errors or warnings
- [x] The PR follows the [Contribution
Guidelines](https://github.com/microsoft/chat-copilot/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/chat-copilot/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
~- [ ] All unit tests pass, and I have added new tests where possible~
- [x] I didn't break anyone 😄

webapi/Auth/PassThroughAuthenticationHandler.cs

@@ -42,4 +42,9 @@ protected override Task<AuthenticateResult> HandleAuthenticateAsync()
 
         return Task.FromResult(AuthenticateResult.Success(ticket));
     }
+
+    /// <summary>
+    /// Returns true if the given user ID is the default user guest ID.
+    /// </summary>
+    public static bool IsDefaultUser(string userId) => userId == DefaultUserId;
 }

webapi/Models/Response/BotResponsePrompt.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft. All rights reserved.
 
 using System.Text.Json.Serialization;
+using ChatCompletionContextMessages = Microsoft.SemanticKernel.AI.ChatCompletion.ChatHistory;
 
 namespace CopilotChat.WebApi.Models.Response;
 
@@ -46,19 +47,13 @@ public class BotResponsePrompt
     public string ChatHistory { get; set; } = string.Empty;
 
     /// <summary>
-    /// Preamble to the LLM's response.
+    /// The collection of context messages associated with this chat completions request.
+    /// See https://learn.microsoft.com/en-us/dotnet/api/azure.ai.openai.chatcompletionsoptions.messages?view=azure-dotnet-preview#azure-ai-openai-chatcompletionsoptions-messages.
     /// </summary>
-    [JsonPropertyName("systemChatContinuation")]
-    public string SystemChatContinuation { get; set; } = string.Empty;
-
-    /// <summary>
-    /// Raw content of the rendered prompt.
-    /// </summary>
-    [JsonPropertyName("rawContent")]
-    public string RawContent { get; set; } = string.Empty;
+    [JsonPropertyName("metaPromptTemplate")]
+    public ChatCompletionContextMessages MetaPromptTemplate { get; set; } = new();
 
     public BotResponsePrompt(
-        string rawContent,
         string systemDescription,
         string systemResponse,
         string audience,
@@ -67,16 +62,15 @@ public class BotResponsePrompt
         string documentMemories,
         SemanticDependency<StepwiseThoughtProcess> externalInformation,
         string chatHistory,
-        string systemChatContinuation
-        )
+        ChatCompletionContextMessages metaPromptTemplate
+    )
     {
-        this.RawContent = rawContent;
         this.SystemPersona = string.Join("\n", systemDescription, systemResponse);
         this.Audience = audience;
         this.UserIntent = userIntent;
         this.PastMemories = string.Join("\n", chatMemories, documentMemories).Trim();
         this.ExternalInformation = externalInformation;
         this.ChatHistory = chatHistory;
-        this.SystemChatContinuation = systemChatContinuation;
+        this.MetaPromptTemplate = metaPromptTemplate;
     }
 }

webapi/Options/PromptsOptions.cs

@@ -138,22 +138,13 @@ public class PromptsOptions
     };
 
     // Chat commands
-    internal string SystemChatContinuation = "SINGLE RESPONSE FROM BOT TO USER:\n[{{TimeSkill.Now}} {{timeSkill.Second}}] bot:";
-
-    // Regex to match system chat continuation preamble in rendered prompt
-    internal const string SYSTEM_CHAT_CONTINUATION_REGEX = @"(SINGLE RESPONSE FROM BOT TO USER:\n\[.*] bot:)";
-
-    internal string[] SystemChatPromptComponents => new string[]
+    internal string[] SystemPersonaComponents => new string[]
     {
         this.SystemDescription,
         this.SystemResponse,
-        "{{$audience}}",
-        "{{$userIntent}}",
-        "{{$chatContext}}",
-        this.SystemChatContinuation
     };
 
-    internal string SystemChatPrompt => string.Join("\n\n", this.SystemChatPromptComponents);
+    internal string SystemPersona => string.Join("\n\n", this.SystemPersonaComponents);
 
     internal double ResponseTemperature { get; } = 0.7;
     internal double ResponseTopP { get; } = 1;

webapi/Skills/ChatSkills/ExternalInformationSkill.cs

@@ -52,11 +52,6 @@ public class ExternalInformationSkill
     /// </summary>
     public StepwiseThoughtProcess? StepwiseThoughtProcess { get; private set; }
 
-    /// <summary>
-    /// Preamble to add to the related information text.
-    /// </summary>
-    private const string PromptPreamble = "[SOURCE START]";
-
     /// <summary>
     /// Supplement to help guide model in using data.
     /// </summary>
@@ -67,11 +62,6 @@ public class ExternalInformationSkill
     /// </summary>
     private const string ResultHeader = "RESULT: ";
 
-    /// <summary>
-    /// Postamble to add to the related information text.
-    /// </summary>
-    private const string PromptPostamble = "[SOURCE END]";
-
     /// <summary>
     /// Create a new instance of ExternalInformationSkill.
     /// </summary>
@@ -104,8 +94,8 @@ public class ExternalInformationSkill
             return string.Empty;
         }
 
-        var contextString = string.Join("\n", context.Variables.Where(v => v.Key != "userIntent").Select(v => $"{v.Key}: {v.Value}"));
-        var goal = $"Given the following context, accomplish the user intent.\nContext:\n{contextString}\nUser Intent:{userIntent}";
+        var contextString = string.Join("\n", context.Variables.Select(v => $"{v.Key}: {v.Value}"));
+        var goal = $"Given the following context, accomplish the user intent.\nContext:\n{contextString}\n{userIntent}";
         if (this._planner.PlannerOptions?.Type == PlanType.Stepwise)
         {
             var plannerContext = context.Clone();
@@ -114,7 +104,7 @@ public class ExternalInformationSkill
                 plannerContext.Variables["stepsTaken"],
                 plannerContext.Variables["timeTaken"],
                 plannerContext.Variables["skillCount"]);
-            return $"{PromptPreamble}\n{plannerContext.Variables.Input.Trim()}\n{PromptPostamble}\n";
+            return $"{plannerContext.Variables.Input.Trim()}\n";
         }
 
         // Check if plan exists in ask's context variables.
@@ -136,8 +126,6 @@ public class ExternalInformationSkill
 
             int tokenLimit =
                 int.Parse(context.Variables["tokenLimit"], new NumberFormatInfo()) -
-                TokenUtilities.TokenCount(PromptPreamble) -
-                TokenUtilities.TokenCount(PromptPostamble) -
                 TokenUtilities.TokenCount(functionsUsed) -
                 TokenUtilities.TokenCount(ResultHeader);
 
@@ -154,7 +142,7 @@ public class ExternalInformationSkill
                 planResult = newPlanContext.Variables.Input;
             }
 
-            return $"{PromptPreamble}\n{PromptSupplement}\n{functionsUsed}\n{ResultHeader}{planResult.Trim()}\n{PromptPostamble}\n";
+            return $"{PromptSupplement}\n{functionsUsed}\n{ResultHeader}{planResult.Trim()}\n";
         }
         else
         {

webapi/Skills/TokenUtilities.cs

@@ -5,9 +5,11 @@
 using System.Globalization;
 using System.Linq;
 using Microsoft.Extensions.Logging;
+using Microsoft.SemanticKernel.AI.ChatCompletion;
 using Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk;
 using Microsoft.SemanticKernel.Connectors.AI.OpenAI.Tokenizers;
 using Microsoft.SemanticKernel.Orchestration;
+using ChatCompletionContextMessages = Microsoft.SemanticKernel.AI.ChatCompletion.ChatHistory;
 
 namespace CopilotChat.WebApi.Skills;
 
@@ -86,5 +88,34 @@ internal static void GetFunctionTokenUsage(SKContext result, SKContext chatConte
     /// <summary>
     /// Calculate the number of tokens in a string.
     /// </summary>
+    /// <param name="text">The string to calculate the number of tokens in.</param>
     internal static int TokenCount(string text) => GPT3Tokenizer.Encode(text).Count;
+
+    /// <summary>
+    /// Rough token costing of ChatHistory's message object.
+    /// Follows the syntax defined by Azure OpenAI's ChatMessage object: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#chatmessage
+    /// e.g., "message": {"role":"assistant","content":"Yes }
+    /// </summary>
+    /// <param name="authorRole">Author role of the message.</param>
+    /// <param name="content">Content of the message.</param>
+    internal static int GetContextMessageTokenCount(AuthorRole authorRole, string content)
+    {
+        var tokenCount = authorRole == AuthorRole.System ? TokenCount("\n") : 0;
+        return tokenCount + TokenCount($"role:{authorRole.Label}") + TokenCount($"content:{content}");
+    }
+
+    /// <summary>
+    /// Rough token costing of ChatCompletionContextMessages object.
+    /// </summary>
+    /// <param name="chatHistory">ChatCompletionContextMessages object to calculate the number of tokens of.</param>
+    internal static int GetContextMessagesTokenCount(ChatCompletionContextMessages chatHistory)
+    {
+        var tokenCount = 0;
+        foreach (var message in chatHistory.Messages)
+        {
+            tokenCount += GetContextMessageTokenCount(message.Role, message.Content);
+        }
+
+        return tokenCount;
+    }
 }

webapi/appsettings.json

@@ -225,7 +225,7 @@
     "CompletionTokenLimit": 4096,
     "ResponseTokenLimit": 1024,
     "SystemDescription": "This is a chat between an intelligent AI bot named Copilot and one or more participants. SK stands for Semantic Kernel, the AI platform used to build the bot. The AI was trained on data through 2021 and is not aware of events that have occurred since then. It also has no ability to access data on the Internet, so it should not claim that it can or say that it will go and look things up. Try to be concise with your answers, though it is not required. Knowledge cutoff: {{$knowledgeCutoff}} / Current date: {{TimeSkill.Now}}.",
-    "SystemResponse": "Either return [silence] or provide a response to the last message. If you provide a response do not provide a list of possible responses or completions, just a single response. ONLY PROVIDE A RESPONSE IF the last message WAS ADDRESSED TO THE 'BOT' OR 'COPILOT'. If it appears the last message was not for you, send [silence] as the bot response.",
+    "SystemResponse": "Either return [silence] or provide a response to the last message. ONLY PROVIDE A RESPONSE IF the last message WAS ADDRESSED TO THE 'BOT' OR 'COPILOT'. If it appears the last message was not for you, send [silence] as the bot response.",
     "InitialBotMessage": "Hello, thank you for democratizing AI's productivity benefits with open source! How can I help you today?",
     "KnowledgeCutoffDate": "Saturday, January 1, 2022",
     "SystemAudience": "Below is a chat history between an intelligent AI bot named Copilot with one or more participants.",

webapp/src/components/chat/plan-viewer/PlanViewer.tsx

@@ -68,7 +68,7 @@ export const PlanViewer: React.FC<PlanViewerProps> = ({ message, messageIndex, g
         originalPlan.steps[0].parameters = originalPlan.parameters;
     }
 
-    const userIntentPrefix = 'User Intent:User intent: ';
+    const userIntentPrefix = 'User intent: ';
     const userIntentIndex = originalPlan.description.indexOf(userIntentPrefix) as number;
     const description: string =
         userIntentIndex !== -1

webapp/src/components/chat/prompt-dialog/PromptDialog.tsx

@@ -10,6 +10,7 @@ import {
     DialogSurface,
     DialogTitle,
     DialogTrigger,
+    Divider,
     Label,
     Link,
     SelectTabEventHandler,
@@ -24,18 +25,31 @@ import {
 import { Info16Regular } from '@fluentui/react-icons';
 import React from 'react';
 import { BotResponsePrompt, DependencyDetails, PromptSectionsNameMap } from '../../../libs/models/BotResponsePrompt';
-import { IChatMessage } from '../../../libs/models/ChatMessage';
+import { ChatMessageType, IChatMessage } from '../../../libs/models/ChatMessage';
 import { PlanType } from '../../../libs/models/Plan';
 import { StepwiseThoughtProcess } from '../../../libs/models/StepwiseThoughtProcess';
-import { useDialogClasses } from '../../../styles';
+import { SharedStyles, useDialogClasses } from '../../../styles';
 import { TokenUsageGraph } from '../../token-usage/TokenUsageGraph';
 import { formatParagraphTextContent } from '../../utils/TextUtils';
 import { StepwiseThoughtProcessView } from './stepwise-planner/StepwiseThoughtProcessView';
 
 const useClasses = makeStyles({
-    prompt: {
+    root: {
+        display: 'flex',
+        flexDirection: 'column',
+        ...shorthands.overflow('hidden'),
+    },
+    outer: {
+        paddingRight: tokens.spacingVerticalXS,
+    },
+    promptDetails: {
         marginTop: tokens.spacingHorizontalS,
     },
+    content: {
+        height: '100%',
+        ...SharedStyles.scroll,
+        paddingRight: tokens.spacingVerticalL,
+    },
     infoButton: {
         ...shorthands.padding(0),
         ...shorthands.margin(0),
@@ -91,8 +105,8 @@ export const PromptDialog: React.FC<IPromptDialogProps> = ({ message }) => {
                 value += '\nNo relevant document memories.';
             }
 
-            return value && key !== 'rawContent' ? (
-                <div className={classes.prompt} key={`prompt-details-${key}`}>
+            return value && key !== 'metaPromptTemplate' ? (
+                <div className={classes.promptDetails} key={`prompt-details-${key}`}>
                     <Body1Strong>{PromptSectionsNameMap[key]}</Body1Strong>
                     {isStepwiseThoughtProcess ? (
                         <StepwiseThoughtProcessView thoughtProcess={value as DependencyDetails} />
@@ -111,10 +125,14 @@ export const PromptDialog: React.FC<IPromptDialogProps> = ({ message }) => {
                     <Button className={classes.infoButton} icon={<Info16Regular />} appearance="transparent" />
                 </Tooltip>
             </DialogTrigger>
-            <DialogSurface>
-                <DialogBody>
+            <DialogSurface className={classes.outer}>
+                <DialogBody
+                    style={{
+                        height: message.type !== ChatMessageType.Message || !message.prompt ? 'fit-content' : '825px',
+                    }}
+                >
                     <DialogTitle>Prompt</DialogTitle>
-                    <DialogContent>
+                    <DialogContent className={classes.root}>
                         <TokenUsageGraph promptView tokenUsage={message.tokenUsage ?? {}} />
                         {message.prompt && typeof prompt !== 'string' && (
                             <TabList selectedValue={selectedTab} onTabSelect={onTabSelect}>
@@ -126,9 +144,19 @@ export const PromptDialog: React.FC<IPromptDialogProps> = ({ message }) => {
                                 </Tab>
                             </TabList>
                         )}
-                        {selectedTab === 'formatted' && promptDetails}
-                        {selectedTab === 'rawContent' &&
-                            formatParagraphTextContent((prompt as BotResponsePrompt).rawContent)}
+                        <div className={message.prompt && typeof prompt !== 'string' ? classes.content : undefined}>
+                            {selectedTab === 'formatted' && promptDetails}
+                            {selectedTab === 'rawContent' &&
+                                (prompt as BotResponsePrompt).metaPromptTemplate.map((contextMessage, index) => {
+                                    return (
+                                        <div key={`context-message-${index}`}>
+                                            <p>{`Role: ${contextMessage.Role.Label}`}</p>
+                                            {formatParagraphTextContent(`Content: ${contextMessage.Content}`)}
+                                            <Divider />
+                                        </div>
+                                    );
+                                })}
+                        </div>
                     </DialogContent>
                     <DialogActions position="start" className={dialogClasses.footer}>
                         <Label size="small" color="brand">

webapp/src/components/token-usage/TokenUsageGraph.tsx

@@ -47,6 +47,9 @@ const useClasses = makeStyles({
     legend: {
         'flex-flow': 'wrap',
     },
+    divider: {
+        width: '97%',
+    },
 });
 
 interface ITokenUsageGraph {
@@ -176,7 +179,7 @@ export const TokenUsageGraph: React.FC<ITokenUsageGraph> = ({ promptView, tokenU
                     </>
                 )}
             </div>
-            <Divider />
+            <Divider className={classes.divider} />
         </>
     );
 };

webapp/src/components/utils/TextUtils.tsx

@@ -51,7 +51,7 @@ export function formatChatTextContent(messageContent: string) {
 /*
  * Formats text containing `\n` or `\r` into paragraphs.
  */
-export function formatParagraphTextContent(messageContent: string) {
+export function formatParagraphTextContent(messageContent = '') {
     messageContent = messageContent.replaceAll('\r\n', '\n\r');
 
     return (

webapp/src/libs/models/BotResponsePrompt.ts

@@ -20,11 +20,9 @@ export interface BotResponsePrompt {
     // Recent messages from history of the conversation.
     chatHistory: string;
 
-    // Preamble to the LLM's response.
-    systemChatContinuation: string;
-
-    // Raw content of the rendered prompt.
-    rawContent: string;
+    // The collection of context messages associated with this chat completions request.
+    // Also serves as the rendered prompt template.
+    metaPromptTemplate: ContextMessage[];
 }
 
 export const PromptSectionsNameMap: Record<string, string> = {
@@ -34,7 +32,6 @@ export const PromptSectionsNameMap: Record<string, string> = {
     chatMemories: 'Chat Memories',
     externalInformation: 'Planner Results',
     chatHistory: 'Chat History',
-    systemChatContinuation: 'System Chat Continuation',
 };
 
 // Information about semantic dependencies of the prompt.
@@ -45,3 +42,22 @@ export interface DependencyDetails {
     // Result of the dependency. This is the output that's injected into the prompt.
     result: string;
 }
+
+// As defined by ChatRole struct in the Azure OpenAI SDK.
+// See https://learn.microsoft.com/en-us/dotnet/api/azure.ai.openai.chatrole?view=azure-dotnet-preview.
+enum AuthorRoles {
+    System = 'System',
+    User = 'User',
+    Assistant = 'Assistant',
+    Tool = 'Tool',
+    Function = 'Function',
+}
+
+// The collection of context messages associated with this chat completions request.
+// See https://learn.microsoft.com/en-us/dotnet/api/azure.ai.openai.chatcompletionsoptions.messages?view=azure-dotnet-preview#azure-ai-openai-chatcompletionsoptions-messages.
+interface ContextMessage {
+    Role: {
+        Label: AuthorRoles;
+    };
+    Content: string;
+}