Skip to content

Commit 3f593a1

Browse files
crickmanwestey-m
andauthored
.NET Agent - Add AIContext to OpenAIResponseAgent (#12456)
### Motivation and Context <!-- Thank you for your contribution to the semantic-kernel repo! Please help reviewers and future users, providing the following information: 1. Why is this change required? 2. What problem does it solve? 3. What scenario does it contribute to? 4. If it fixes an open issue, please link to the issue here. --> Follow-up to #12444 Fixes: #12472 ### Description <!-- Describe your changes, the overall approach, the underlying design. These notes will help understanding how your code works. Thanks! --> Add support for incorporating `AIContext` when invoking `OpenAIResponseAgent` Enabled conformance tests ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [X] The code builds clean without any errors or warnings - [X] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [X] All unit tests pass, and I have added new tests where possible - [X] I didn't break anyone 😄 --------- Co-authored-by: westey <164392973+westey-m@users.noreply.github.com>
1 parent 4c71386 commit 3f593a1

File tree

13 files changed

+299
-65
lines changed

13 files changed

+299
-65
lines changed
Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using Azure.AI.OpenAI;
4+
using Azure.Identity;
5+
using Microsoft.Extensions.AI;
6+
using Microsoft.SemanticKernel;
7+
using Microsoft.SemanticKernel.Agents;
8+
using Microsoft.SemanticKernel.Agents.OpenAI;
9+
using Microsoft.SemanticKernel.ChatCompletion;
10+
using Microsoft.SemanticKernel.Memory;
11+
12+
namespace Agents;
13+
14+
#pragma warning disable SKEXP0130 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
15+
16+
/// <summary>
17+
/// Demonstrate creation of <see cref="OpenAIResponseAgent"/> and
18+
/// adding whiteboarding capabilities, where the most relevant information from the conversation is captured on a whiteboard.
19+
/// This is useful for long running conversations where the conversation history may need to be truncated
20+
/// over time, but you do not want to agent to lose context.
21+
/// </summary>
22+
public class OpenAIResponseAgent_Whiteboard(ITestOutputHelper output) : BaseResponsesAgentTest(output)
23+
{
24+
private const string AgentName = "FriendlyAssistant";
25+
private const string AgentInstructions = "You are a friendly assistant";
26+
27+
/// <summary>
28+
/// Shows how to allow an agent to use a whiteboard for storing the most important information
29+
/// from a long running, truncated conversation.
30+
/// </summary>
31+
[Fact]
32+
private async Task UseWhiteboardForShortTermMemory()
33+
{
34+
IChatClient chatClient = new AzureOpenAIClient(new Uri(TestConfiguration.AzureOpenAI.Endpoint), new AzureCliCredential())
35+
.GetChatClient(TestConfiguration.AzureOpenAI.ChatDeploymentName)
36+
.AsIChatClient();
37+
38+
// Create the whiteboard.
39+
WhiteboardProvider whiteboardProvider = new(chatClient);
40+
41+
OpenAIResponseAgent agent = new(this.Client)
42+
{
43+
Name = AgentName,
44+
Instructions = AgentInstructions,
45+
Arguments = new KernelArguments(new PromptExecutionSettings { FunctionChoiceBehavior = FunctionChoiceBehavior.Auto() }),
46+
StoreEnabled = false,
47+
};
48+
49+
// Create the agent with our sample plugin.
50+
agent.Kernel.Plugins.AddFromType<VMPlugin>();
51+
52+
// Create a chat history reducer that we can use to truncate the chat history
53+
// when it goes over 3 items.
54+
ChatHistoryTruncationReducer chatHistoryReducer = new(3, 3);
55+
56+
// Create a thread for the agent and add the whiteboard to it.
57+
ChatHistoryAgentThread agentThread = new();
58+
agentThread.AIContextProviders.Add(whiteboardProvider);
59+
60+
// Simulate a conversation with the agent.
61+
// We will also truncate the conversation once it goes over a few items.
62+
await InvokeWithConsoleWriteLine("Hello");
63+
await InvokeWithConsoleWriteLine("I'd like to create a VM?");
64+
await InvokeWithConsoleWriteLine("I want it to have 3 cores.");
65+
await InvokeWithConsoleWriteLine("I want it to have 48GB of RAM.");
66+
await InvokeWithConsoleWriteLine("I want it to have a 500GB Harddrive.");
67+
await InvokeWithConsoleWriteLine("I want it in Europe.");
68+
await InvokeWithConsoleWriteLine("Can you make it Linux and call it 'ContosoVM'.");
69+
await InvokeWithConsoleWriteLine("OK, let's call it `ContosoFinanceVM_Europe` instead.");
70+
await InvokeWithConsoleWriteLine("Thanks, now I want to create another VM.");
71+
await InvokeWithConsoleWriteLine("Make all the options the same as the last one, except for the region, which should be North America, and the name, which should be 'ContosoFinanceVM_NorthAmerica'.");
72+
73+
async Task InvokeWithConsoleWriteLine(string message)
74+
{
75+
// Print the user input.
76+
Console.WriteLine($"User: {message}");
77+
78+
// Invoke the agent.
79+
ChatMessageContent response = await agent.InvokeAsync(message, agentThread).FirstAsync();
80+
81+
// Print the response.
82+
this.WriteAgentChatMessage(response);
83+
84+
// Make sure any async whiteboard processing is complete before we print out its contents.
85+
await whiteboardProvider.WhenProcessingCompleteAsync();
86+
87+
// Print out the whiteboard contents.
88+
Console.WriteLine("Whiteboard contents:");
89+
foreach (var item in whiteboardProvider.CurrentWhiteboardContent)
90+
{
91+
Console.WriteLine($"- {item}");
92+
}
93+
Console.WriteLine();
94+
95+
// Truncate the chat history if it gets too big.
96+
await agentThread.ChatHistory.ReduceInPlaceAsync(chatHistoryReducer, CancellationToken.None);
97+
}
98+
}
99+
100+
private sealed class VMPlugin
101+
{
102+
[KernelFunction]
103+
public Task<VMCreateResult> CreateVM(Region region, OperatingSystem os, string name, int numberOfCores, int memorySizeInGB, int hddSizeInGB)
104+
{
105+
if (name == "ContosoVM")
106+
{
107+
throw new Exception("VM name already exists");
108+
}
109+
110+
return Task.FromResult(new VMCreateResult { VMId = Guid.NewGuid().ToString() });
111+
}
112+
}
113+
114+
public class VMCreateResult
115+
{
116+
public string VMId { get; set; } = string.Empty;
117+
}
118+
119+
private enum Region
120+
{
121+
NorthAmerica,
122+
SouthAmerica,
123+
Europe,
124+
Asia,
125+
Africa,
126+
Australia
127+
}
128+
129+
private enum OperatingSystem
130+
{
131+
Windows,
132+
Linux,
133+
MacOS
134+
}
135+
}

dotnet/src/Agents/OpenAI/Extensions/ChatContentMessageExtensions.cs

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,12 +43,13 @@ public static IEnumerable<ThreadInitializationMessage> ToThreadInitializationMes
4343
/// <returns>A <see cref="ResponseItem"/> instance.</returns>
4444
public static ResponseItem ToResponseItem(this ChatMessageContent message)
4545
{
46+
string content = message.Content ?? string.Empty;
4647
return message.Role.Label switch
4748
{
48-
"system" => ResponseItem.CreateSystemMessageItem(message.Content),
49-
"user" => ResponseItem.CreateUserMessageItem(message.Content),
50-
"developer" => ResponseItem.CreateDeveloperMessageItem(message.Content),
51-
"assistant" => ResponseItem.CreateAssistantMessageItem(message.Content),
49+
"system" => ResponseItem.CreateSystemMessageItem(content),
50+
"user" => ResponseItem.CreateUserMessageItem(content),
51+
"developer" => ResponseItem.CreateDeveloperMessageItem(content),
52+
"assistant" => ResponseItem.CreateAssistantMessageItem(content),
5253
_ => throw new NotSupportedException($"Unsupported role {message.Role.Label}. Only system, user, developer or assistant roles are allowed."),
5354
};
5455
}

dotnet/src/Agents/OpenAI/Internal/ResponseCreationOptionsFactory.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,10 @@ internal static ResponseCreationOptions CreateOptions(
2323
return responseAgentInvokeOptions.ResponseCreationOptions;
2424
}
2525

26-
var responseTools = agent.Kernel.Plugins
26+
var responseTools = agent.GetKernel(invokeOptions).Plugins
2727
.SelectMany(kp => kp.Select(kf => kf.ToResponseTool(kp.Name)));
2828

29-
var creationOptions = new ResponseCreationOptions()
29+
var creationOptions = new ResponseCreationOptions
3030
{
3131
EndUserId = agent.GetDisplayName(),
3232
Instructions = $"{agent.Instructions}\n{invokeOptions?.AdditionalInstructions}",

dotnet/src/Agents/OpenAI/Internal/ResponseThreadActions.cs

Lines changed: 23 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
using System.Text;
99
using System.Threading;
1010
using System.Threading.Tasks;
11+
using Microsoft.SemanticKernel.Agents.Extensions;
1112
using Microsoft.SemanticKernel.ChatCompletion;
1213
using Microsoft.SemanticKernel.Connectors.FunctionCalling;
1314
using OpenAI.Responses;
@@ -27,14 +28,13 @@ internal static async IAsyncEnumerable<ChatMessageContent> InvokeAsync(
2728
AgentInvokeOptions options,
2829
[EnumeratorCancellation] CancellationToken cancellationToken)
2930
{
30-
var kernel = options?.Kernel ?? agent.Kernel;
3131
var responseAgentThread = agentThread as OpenAIResponseAgentThread;
3232

3333
var overrideHistory = history;
3434
if (!agent.StoreEnabled)
3535
{
3636
// Use the thread chat history
37-
overrideHistory = [.. GetChatHistory(agentThread, history)];
37+
overrideHistory = [.. GetChatHistory(agentThread)];
3838
}
3939

4040
var creationOptions = ResponseCreationOptionsFactory.CreateOptions(agent, agentThread, options);
@@ -86,7 +86,7 @@ await functionProcessor.InvokeFunctionCallsAsync(
8686
message,
8787
(_) => true,
8888
functionOptions,
89-
kernel,
89+
agent.GetKernel(options),
9090
isStreaming: false,
9191
cancellationToken).ToArrayAsync(cancellationToken).ConfigureAwait(false);
9292
var functionOutputItems = functionResults.Select(fr => ResponseItem.CreateFunctionCallOutputItem(fr.CallId, fr.Result?.ToString() ?? string.Empty)).ToList();
@@ -102,8 +102,7 @@ await functionProcessor.InvokeFunctionCallsAsync(
102102
}
103103

104104
// Return the function results as a message
105-
ChatMessageContentItemCollection items = new();
106-
items.AddRange(functionResults);
105+
ChatMessageContentItemCollection items = [.. functionResults];
107106
ChatMessageContent functionResultMessage = new()
108107
{
109108
Role = AuthorRole.Tool,
@@ -121,14 +120,13 @@ internal static async IAsyncEnumerable<StreamingChatMessageContent> InvokeStream
121120
AgentInvokeOptions? options,
122121
[EnumeratorCancellation] CancellationToken cancellationToken)
123122
{
124-
var kernel = options?.Kernel ?? agent.Kernel;
125123
var responseAgentThread = agentThread as OpenAIResponseAgentThread;
126124

127125
var overrideHistory = history;
128126
if (!agent.StoreEnabled)
129127
{
130128
// Use the thread chat history
131-
overrideHistory = [.. GetChatHistory(agentThread, history)];
129+
overrideHistory = [.. GetChatHistory(agentThread)];
132130
}
133131

134132
var inputItems = overrideHistory.Select(m => m.ToResponseItem()).ToList();
@@ -161,6 +159,7 @@ internal static async IAsyncEnumerable<StreamingChatMessageContent> InvokeStream
161159
case StreamingResponseCompletedUpdate completedUpdate:
162160
response = completedUpdate.Response;
163161
message = completedUpdate.Response.ToChatMessageContent();
162+
overrideHistory.Add(message);
164163
break;
165164

166165
case StreamingResponseOutputItemAddedUpdate outputItemAddedUpdate:
@@ -258,8 +257,8 @@ await functionProcessor.InvokeFunctionCallsAsync(
258257
message!,
259258
(_) => true,
260259
functionOptions,
261-
kernel,
262-
isStreaming: false,
260+
agent.GetKernel(options),
261+
isStreaming: true,
263262
cancellationToken).ToArrayAsync(cancellationToken).ConfigureAwait(false);
264263
var functionOutputItems = functionResults.Select(fr => ResponseItem.CreateFunctionCallOutputItem(fr.CallId, fr.Result?.ToString() ?? string.Empty)).ToList();
265264

@@ -274,21 +273,26 @@ await functionProcessor.InvokeFunctionCallsAsync(
274273
}
275274

276275
// Return the function results as a message
277-
ChatMessageContentItemCollection items = new();
278-
items.AddRange(functionResults);
279-
StreamingChatMessageContent functionResultMessage = new(
280-
AuthorRole.Tool,
281-
content: null)
276+
ChatMessageContentItemCollection items = [.. functionResults];
277+
ChatMessageContent functionResultMessage = new()
282278
{
283-
ModelId = modelId,
284-
InnerContent = functionCallUpdateContent,
285-
Items = [functionCallUpdateContent],
279+
Role = AuthorRole.Tool,
280+
Items = items,
286281
};
287-
yield return functionResultMessage;
282+
StreamingChatMessageContent streamingFunctionResultMessage =
283+
new(AuthorRole.Tool,
284+
content: null)
285+
{
286+
ModelId = modelId,
287+
InnerContent = functionCallUpdateContent,
288+
Items = [functionCallUpdateContent],
289+
};
290+
overrideHistory.Add(functionResultMessage);
291+
yield return streamingFunctionResultMessage;
288292
}
289293
}
290294

291-
private static ChatHistory GetChatHistory(AgentThread agentThread, ChatHistory history)
295+
private static ChatHistory GetChatHistory(AgentThread agentThread)
292296
{
293297
if (agentThread is ChatHistoryAgentThread chatHistoryAgentThread)
294298
{
@@ -298,17 +302,6 @@ private static ChatHistory GetChatHistory(AgentThread agentThread, ChatHistory h
298302
throw new InvalidOperationException("The agent thread is not a ChatHistoryAgentThread.");
299303
}
300304

301-
private static void UpdateResponseId(AgentThread agentThread, string id)
302-
{
303-
if (agentThread is OpenAIResponseAgentThread openAIResponseAgentThread)
304-
{
305-
openAIResponseAgentThread.ResponseId = id;
306-
return;
307-
}
308-
309-
throw new InvalidOperationException("The agent thread is not an OpenAIResponseAgentThread.");
310-
}
311-
312305
private static void ThrowIfIncompleteOrFailed(OpenAIResponseAgent agent, OpenAIResponse response)
313306
{
314307
if (response.Status == ResponseStatus.Incomplete || response.Status == ResponseStatus.Failed)

0 commit comments

Comments
 (0)