[.Net] Bring Semantic-Kernel to AutoGen && Propose a new message abstraction to support multi-modal/parallel function call && potential breaking change from openai api

dotnet/.editorconfig

@@ -167,9 +167,6 @@ dotnet_diagnostic.IDE0005.severity = error
 # IDE0069: Remove unused local variable
 dotnet_diagnostic.IDE0069.severity = error
 
-# IDE0060: Remove unused parameter
-dotnet_diagnostic.IDE0060.severity = warning
-
 # disable CS1573: Parameter has no matching param tag in the XML comment for
 dotnet_diagnostic.CS1573.severity = none
 

dotnet/AutoGen.sln

@@ -21,7 +21,13 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "sample", "sample", "{FBFEAD
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.DotnetInteractive", "src\AutoGen.DotnetInteractive\AutoGen.DotnetInteractive.csproj", "{B61D8008-7FB7-4C0E-8044-3A74AA63A596}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AutoGen.LMStudio", "src\AutoGen.LMStudio\AutoGen.LMStudio.csproj", "{F98BDA9B-8657-4BA8-9B03-BAEA454CAE60}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.LMStudio", "src\AutoGen.LMStudio\AutoGen.LMStudio.csproj", "{F98BDA9B-8657-4BA8-9B03-BAEA454CAE60}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AutoGen.SemanticKernel", "src\AutoGen.SemanticKernel\AutoGen.SemanticKernel.csproj", "{45D6FC80-36F3-4967-9663-E20B63824621}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AutoGen.Core", "src\AutoGen.Core\AutoGen.Core.csproj", "{D58D43D1-0617-4A3D-9932-C773E6398535}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AutoGen.OpenAI", "src\AutoGen.OpenAI\AutoGen.OpenAI.csproj", "{63445BB7-DBB9-4AEF-9D6F-98BBE75EE1EC}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -57,6 +63,18 @@ Global
 		{F98BDA9B-8657-4BA8-9B03-BAEA454CAE60}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{F98BDA9B-8657-4BA8-9B03-BAEA454CAE60}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{F98BDA9B-8657-4BA8-9B03-BAEA454CAE60}.Release|Any CPU.Build.0 = Release|Any CPU
+		{45D6FC80-36F3-4967-9663-E20B63824621}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{45D6FC80-36F3-4967-9663-E20B63824621}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{45D6FC80-36F3-4967-9663-E20B63824621}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{45D6FC80-36F3-4967-9663-E20B63824621}.Release|Any CPU.Build.0 = Release|Any CPU
+		{D58D43D1-0617-4A3D-9932-C773E6398535}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{D58D43D1-0617-4A3D-9932-C773E6398535}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{D58D43D1-0617-4A3D-9932-C773E6398535}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{D58D43D1-0617-4A3D-9932-C773E6398535}.Release|Any CPU.Build.0 = Release|Any CPU
+		{63445BB7-DBB9-4AEF-9D6F-98BBE75EE1EC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{63445BB7-DBB9-4AEF-9D6F-98BBE75EE1EC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{63445BB7-DBB9-4AEF-9D6F-98BBE75EE1EC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{63445BB7-DBB9-4AEF-9D6F-98BBE75EE1EC}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -69,6 +87,9 @@ Global
 		{7EBF916A-A7B1-4B74-AF10-D705B7A18F58} = {FBFEAD1F-29EB-4D99-A672-0CD8473E10B9}
 		{B61D8008-7FB7-4C0E-8044-3A74AA63A596} = {18BF8DD7-0585-48BF-8F97-AD333080CE06}
 		{F98BDA9B-8657-4BA8-9B03-BAEA454CAE60} = {18BF8DD7-0585-48BF-8F97-AD333080CE06}
+		{45D6FC80-36F3-4967-9663-E20B63824621} = {18BF8DD7-0585-48BF-8F97-AD333080CE06}
+		{D58D43D1-0617-4A3D-9932-C773E6398535} = {18BF8DD7-0585-48BF-8F97-AD333080CE06}
+		{63445BB7-DBB9-4AEF-9D6F-98BBE75EE1EC} = {18BF8DD7-0585-48BF-8F97-AD333080CE06}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {93384647-528D-46C8-922C-8DB36A382F0B}

dotnet/Directory.Build.props

@@ -13,6 +13,8 @@
     <NoWarn>$(NoWarn);$(CSNoWarn);NU5104</NoWarn>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
     <IsPackable>false</IsPackable>
+    <EnableNetAnalyzers>true</EnableNetAnalyzers>
+    <EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
   </PropertyGroup>
 
   <PropertyGroup>

dotnet/eng/MetaInfo.props

@@ -1,9 +1,9 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
     <PropertyGroup>
-        <VersionPrefix>0.0.8</VersionPrefix>
+        <VersionPrefix>0.0.9</VersionPrefix>
         <Authors>AutoGen</Authors>
-        <PackageProjectUrl>https://github.com/microsoft/autogen</PackageProjectUrl>
+        <PackageProjectUrl>https://microsoft.github.io/autogen-for-net/</PackageProjectUrl>
         <RepositoryUrl>https://github.com/microsoft/autogen</RepositoryUrl>
         <RepositoryType>git</RepositoryType>
         <PackageLicenseExpression>MIT</PackageLicenseExpression>

dotnet/eng/Version.props

@@ -1,11 +1,12 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
     <PropertyGroup>
-        <AzureOpenAIVersion>1.0.0-beta.12</AzureOpenAIVersion>
-        <SemanticKernelVersion>1.0.1</SemanticKernelVersion>
+        <AzureOpenAIVersion>1.0.0-beta.13</AzureOpenAIVersion>
+        <SemanticKernelVersion>1.5.0</SemanticKernelVersion>
+        <SemanticKernelExperimentalVersion>1.5.0-alpha</SemanticKernelExperimentalVersion>
         <SystemCodeDomVersion>5.0.0</SystemCodeDomVersion>
         <MicrosoftCodeAnalysisVersion>4.3.0</MicrosoftCodeAnalysisVersion>
-        <ApprovalTestVersion>5.2.4</ApprovalTestVersion>
+        <ApprovalTestVersion>6.0.0</ApprovalTestVersion>
         <FluentAssertionVersion>6.8.0</FluentAssertionVersion>
         <XUnitVersion>2.4.2</XUnitVersion>
         <MicrosoftNETTestSdkVersion>17.7.0</MicrosoftNETTestSdkVersion>

dotnet/global.json

@@ -1,6 +1,6 @@
 {
   "sdk": {
     "version": "8.0.101",
-    "rollForward": "latestMajor"
+    "rollForward": "latestMinor"
   }
 }

dotnet/nuget/NUGET.md

@@ -1,4 +1,9 @@
 ## AutoGen
+#### Update on 0.0.9 (2024-03-02)
+- Refactor over @AutoGen.Message and introducing `TextMessage`, `ImageMessage`, `MultiModalMessage` and so on. PR [#1676](https://github.com/microsoft/autogen/pull/1676)
+- Add `AutoGen.SemanticKernel` to support seamless integration with Semantic Kernel
+- Move the agent contract abstraction to `AutoGen.Core` package. The `AutoGen.Core` package provides the abstraction for message type, agent and group chat and doesn't contain dependencies over `Azure.AI.OpenAI` or `Semantic Kernel`. This is useful when you want to leverage AutoGen's abstraction only and want to avoid introducing any other dependencies.
+- Move `GPTAgent`, `OpenAIChatAgent` and all openai-dependencies to `AutoGen.OpenAI`
 #### Update on 0.0.8 (2024-02-28)
 - Fix [#1804](https://github.com/microsoft/autogen/pull/1804)
 - Streaming support for IAgent [#1656](https://github.com/microsoft/autogen/pull/1656)

dotnet/sample/AutoGen.BasicSamples/AutoGen.BasicSample.csproj

@@ -6,14 +6,14 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <GenerateDocumentationFile>True</GenerateDocumentationFile>
-    <NoWarn>$(NoWarn);CS8981;CS8600;CS8602;CS8604;CS8618;CS0219</NoWarn>
+    <NoWarn>$(NoWarn);CS8981;CS8600;CS8602;CS8604;CS8618;CS0219;SKEXP0054</NoWarn>
   </PropertyGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\..\src\AutoGen.DotnetInteractive\AutoGen.DotnetInteractive.csproj" />
-    <ProjectReference Include="..\..\src\AutoGen.LMStudio\AutoGen.LMStudio.csproj" />
     <ProjectReference Include="..\..\src\AutoGen.SourceGenerator\AutoGen.SourceGenerator.csproj" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
     <ProjectReference Include="..\..\src\AutoGen\AutoGen.csproj" />
     <PackageReference Include="FluentAssertions" Version="$(FluentAssertionVersion)" />
+    <PackageReference Include="Microsoft.SemanticKernel.Plugins.Web" Version="$(SemanticKernelExperimentalVersion)" />
   </ItemGroup>
 </Project>

dotnet/sample/AutoGen.BasicSamples/CodeSnippet/CreateAnAgent.cs

@@ -86,15 +86,17 @@ public async Task CodeSnippet4()
                 {
                     llmConfig
                 },
-                FunctionDefinitions = new[]
+                FunctionContracts = new[]
                 {
-                    this.UpperCaseFunction, // The FunctionDefinition object for the UpperCase function
+                    this.UpperCaseFunctionContract, // The FunctionDefinition object for the UpperCase function
                 },
             });
 
         var response = await assistantAgent.SendAsync("hello");
-        response.FunctionName.Should().Be("UpperCase");
-        response.Content.Should().BeNullOrEmpty();
+        response.Should().BeOfType<ToolCallMessage>();
+        var toolCallMessage = (ToolCallMessage)response;
+        toolCallMessage.ToolCalls.Count().Should().Be(1);
+        toolCallMessage.ToolCalls.First().FunctionName.Should().Be("UpperCase");
         #endregion code_snippet_4
     }
 
@@ -119,9 +121,9 @@ public async Task CodeSnippet5()
                 {
                     llmConfig
                 },
-                FunctionDefinitions = new[]
+                FunctionContracts = new[]
                 {
-                    this.UpperCaseFunction, // The FunctionDefinition object for the UpperCase function
+                    this.UpperCaseFunctionContract, // The FunctionDefinition object for the UpperCase function
                 },
             },
             functionMap: new Dictionary<string, Func<string, Task<string>>>
@@ -130,7 +132,10 @@ public async Task CodeSnippet5()
             });
 
         var response = await assistantAgent.SendAsync("hello");
-        response.Content.Should().Be("HELLO");
+        response.Should().BeOfType<TextMessage>();
+        response.From.Should().Be("assistant");
+        var textMessage = (TextMessage)response;
+        textMessage.Content.Should().Be("HELLO");
         #endregion code_snippet_5
     }
 }

dotnet/sample/AutoGen.BasicSamples/CodeSnippet/FunctionCallCodeSnippet.cs

@@ -3,7 +3,6 @@
 
 using AutoGen;
 using AutoGen.OpenAI;
-using AutoGen.OpenAI.Extension;
 using FluentAssertions;
 
 public partial class FunctionCallCodeSnippet
@@ -30,15 +29,18 @@ public async Task CodeSnippet4()
                 {
                     llmConfig
                 },
-                FunctionDefinitions = new[]
+                FunctionContracts = new[]
                 {
-                    function.WeatherReportFunctionContract.ToOpenAIFunctionDefinition(), // The FunctionDefinition object for the weather report function
+                    function.WeatherReportFunctionContract,
                 },
             });
 
         var response = await assistantAgent.SendAsync("hello What's the weather in Seattle today? today is 2024-01-01");
-        response.FunctionName.Should().Be("WeatherReport");
-        response.FunctionArguments.Should().Be(@"{""location"":""Seattle"",""date"":""2024-01-01""}");
+        response.Should().BeOfType<ToolCallMessage>();
+        var toolCallMessage = (ToolCallMessage)response;
+        toolCallMessage.ToolCalls.Count().Should().Be(1);
+        toolCallMessage.ToolCalls[0].FunctionName.Should().Be("WeatherReport");
+        toolCallMessage.ToolCalls[0].FunctionArguments.Should().Be(@"{""location"":""Seattle"",""date"":""2024-01-01""}");
         #endregion code_snippet_4
     }
 
@@ -64,9 +66,9 @@ public async Task CodeSnippet6()
                 {
                     llmConfig
                 },
-                FunctionDefinitions = new[]
+                FunctionContracts = new[]
                 {
-                    function.WeatherReportFunctionContract.ToOpenAIFunctionDefinition(), // The FunctionDefinition object for the weather report function
+                    function.WeatherReportFunctionContract,
                 },
             },
             functionMap: new Dictionary<string, Func<string, Task<string>>>
@@ -78,7 +80,9 @@ public async Task CodeSnippet6()
 
         #region code_snippet_6_1
         var response = await assistantAgent.SendAsync("What's the weather in Seattle today? today is 2024-01-01");
-        response.Content.Should().Be("Weather report for Seattle on 2024-01-01 is sunny");
+        response.Should().BeOfType<TextMessage>();
+        var textMessage = (TextMessage)response;
+        textMessage.Content.Should().Be("Weather report for Seattle on 2024-01-01 is sunny");
         #endregion code_snippet_6_1
     }
 
@@ -95,9 +99,9 @@ public async Task TwoAgentWeatherChatTestAsync()
             llmConfig: new ConversableAgentConfig
             {
                 ConfigList = new[] { config },
-                FunctionDefinitions = new[]
+                FunctionContracts = new[]
                 {
-                    function.WeatherReportFunctionContract.ToOpenAIFunctionDefinition(),
+                    function.WeatherReportFunctionContract,
                 },
             });
 
@@ -111,4 +115,4 @@ public async Task TwoAgentWeatherChatTestAsync()
         await user.InitiateChatAsync(assistant, "what's weather in Seattle today, today is 2024-01-01", 10);
         #endregion two_agent_weather_chat
     }
-}
+}

dotnet/sample/AutoGen.BasicSamples/CodeSnippet/MiddlewareAgentCodeSnippet.cs

@@ -20,54 +20,56 @@ public async Task CodeSnippet1()
         // Since no middleware is added, middlewareAgent will simply proxy into the inner agent to generate reply.
         var reply = await middlewareAgent.SendAsync("Hello World");
         reply.From.Should().Be("assistant");
-        reply.Content.Should().Be("Hello World");
+        reply.GetContent().Should().Be("Hello World");
         #endregion code_snippet_1
 
         #region code_snippet_2
         middlewareAgent.Use(async (messages, options, agent, ct) =>
         {
-            var lastMessage = messages.Last();
+            var lastMessage = messages.Last() as TextMessage;
             lastMessage.Content = $"[middleware 0] {lastMessage.Content}";
             return await agent.GenerateReplyAsync(messages, options, ct);
         });
 
         reply = await middlewareAgent.SendAsync("Hello World");
-        reply.Content.Should().Be("[middleware 0] Hello World");
+        reply.Should().BeOfType<TextMessage>();
+        var textReply = (TextMessage)reply;
+        textReply.Content.Should().Be("[middleware 0] Hello World");
         #endregion code_snippet_2
         #region code_snippet_2_1
         middlewareAgent = agent.RegisterMiddleware(async (messages, options, agnet, ct) =>
         {
-            var lastMessage = messages.Last();
+            var lastMessage = messages.Last() as TextMessage;
             lastMessage.Content = $"[middleware 0] {lastMessage.Content}";
             return await agent.GenerateReplyAsync(messages, options, ct);
         });
 
         reply = await middlewareAgent.SendAsync("Hello World");
-        reply.Content.Should().Be("[middleware 0] Hello World");
+        reply.GetContent().Should().Be("[middleware 0] Hello World");
         #endregion code_snippet_2_1
         #region code_snippet_3
         middlewareAgent.Use(async (messages, options, agent, ct) =>
         {
-            var lastMessage = messages.Last();
+            var lastMessage = messages.Last() as TextMessage;
             lastMessage.Content = $"[middleware 1] {lastMessage.Content}";
             return await agent.GenerateReplyAsync(messages, options, ct);
         });
 
         reply = await middlewareAgent.SendAsync("Hello World");
-        reply.Content.Should().Be("[middleware 0] [middleware 1] Hello World");
+        reply.GetContent().Should().Be("[middleware 0] [middleware 1] Hello World");
         #endregion code_snippet_3
 
         #region code_snippet_4
         middlewareAgent.Use(async (messages, options, next, ct) =>
         {
-            var lastMessage = messages.Last();
+            var lastMessage = messages.Last() as TextMessage;
             lastMessage.Content = $"[middleware shortcut]";
 
             return lastMessage;
         });
 
         reply = await middlewareAgent.SendAsync("Hello World");
-        reply.Content.Should().Be("[middleware shortcut]");
+        reply.GetContent().Should().Be("[middleware shortcut]");
         #endregion code_snippet_4
 
         #region retrieve_inner_agent
@@ -92,7 +94,7 @@ public async Task CodeSnippet1()
             var reply = await agent.GenerateReplyAsync(messages, options, ct);
             while (maxAttempt-- > 0)
             {
-                if (JsonSerializer.Deserialize<Dictionary<string, object>>(reply.Content) is { } dict)
+                if (JsonSerializer.Deserialize<Dictionary<string, object>>(reply.GetContent()) is { } dict)
                 {
                     return reply;
                 }

dotnet/sample/AutoGen.BasicSamples/CodeSnippet/TypeSafeFunctionCallCodeSnippet.cs

@@ -2,7 +2,6 @@
 // TypeSafeFunctionCallCodeSnippet.cs
 
 using System.Text.Json;
-using AutoGen;
 using AutoGen.OpenAI.Extension;
 using Azure.AI.OpenAI;
 

dotnet/sample/AutoGen.BasicSamples/Example01_AssistantAgent.cs

@@ -2,8 +2,8 @@
 // Example01_AssistantAgent.cs
 
 using AutoGen;
+using AutoGen.BasicSample;
 using FluentAssertions;
-using autogen = AutoGen.LLMConfigAPI;
 
 /// <summary>
 /// This example shows the basic usage of <see cref="ConversableAgent"/> class.
@@ -12,13 +12,11 @@ public static class Example01_AssistantAgent
 {
     public static async Task RunAsync()
     {
-        // get OpenAI Key and create config
-        var openAIKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY") ?? throw new Exception("Please set OPENAI_API_KEY environment variable.");
-        var llmConfig = autogen.GetOpenAIConfigList(openAIKey, new[] { "gpt-3.5-turbo" });
+        var gpt35 = LLMConfiguration.GetAzureOpenAIGPT3_5_Turbo();
         var config = new ConversableAgentConfig
         {
             Temperature = 0,
-            ConfigList = llmConfig,
+            ConfigList = [gpt35],
         };
 
         // create assistant agent
@@ -30,16 +28,18 @@ public static async Task RunAsync()
 
         // talk to the assistant agent
         var reply = await assistantAgent.SendAsync("hello world");
-        reply.Content?.Should().Be("HELLO WORLD");
+        reply.Should().BeOfType<TextMessage>();
+        reply.GetContent().Should().Be("HELLO WORLD");
 
         // to carry on the conversation, pass the previous conversation history to the next call
-        var conversationHistory = new List<Message>
+        var conversationHistory = new List<IMessage>
         {
-            new Message(Role.User, "hello world"), // first message
+            new TextMessage(Role.User, "hello world"), // first message
             reply, // reply from assistant agent
         };
 
         reply = await assistantAgent.SendAsync("hello world again", conversationHistory);
-        reply.Content?.Should().Be("HELLO WORLD AGAIN");
+        reply.Should().BeOfType<TextMessage>();
+        reply.GetContent().Should().Be("HELLO WORLD AGAIN");
     }
 }