-
Notifications
You must be signed in to change notification settings - Fork 3.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
.Net Enable Usage of Custom Compatible Chat Message API Endpoints wit…
…h OpenAI Connector + Examples (#4753) ### Motivation and Context - Allow usage of custom Message API (OpenAI ChatCompletion Standard) compliant endpoints with the OpenAI Connector. - Refactoring of OpenAI Models and Classes Structure - Adding Examples on using the current changes against `LMStudio`, `Ollama` and `LocalAI` Message APIs. --------- Co-authored-by: Mark Wallace <127216156+markwallace-microsoft@users.noreply.github.com>
- Loading branch information
1 parent
e416946
commit beef63c
Showing
10 changed files
with
318 additions
and
10 deletions.
There are no files selected for viewing
103 changes: 103 additions & 0 deletions
103
dotnet/samples/KernelSyntaxExamples/Example88_CustomMessageAPIEndpoint.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System; | ||
| using System.Threading.Tasks; | ||
| using Microsoft.SemanticKernel; | ||
| using Microsoft.SemanticKernel.Connectors.OpenAI; | ||
| using Xunit; | ||
| using Xunit.Abstractions; | ||
|
|
||
| namespace Examples; | ||
|
|
||
| /// <summary> | ||
| /// This example shows a way of using OpenAI connector with other APIs that supports the same ChatCompletion Message API standard from OpenAI. | ||
| /// | ||
| /// To proceed with this example will be necessary to follow those steps: | ||
| /// 1. Install LMStudio Platform in your environment | ||
| /// 2. Open LM Studio | ||
| /// 3. Search and Download both Phi2 and Llama2 models (preferably the ones that uses 8GB RAM or more) | ||
| /// 4. Start the Message API Server on http://localhost:1234 | ||
| /// 5. Run the examples. | ||
| /// | ||
| /// OR | ||
| /// | ||
| /// 1. Start the Ollama Message API Server on http://localhost:11434 using docker | ||
| /// 2. docker run -d --gpus=all -v "d:\temp\ollama:/root/.ollama" -p 11434:11434 --name ollama ollama/ollama <see href="https://ollama.com/blog/ollama-is-now-available-as-an-official-docker-image" /> | ||
| /// 3. Set Llama2 as the current ollama model: docker exec -it ollama ollama run llama2 | ||
| /// 4. Run the Ollama examples. | ||
| /// | ||
| /// OR | ||
| /// | ||
| /// 1. Start the LocalAI Message API Server on http://localhost:8080 | ||
| /// 2. docker run -ti -p 8080:8080 localai/localai:v2.12.3-ffmpeg-core phi-2 <see href="https://localai.io/docs/getting-started/run-other-models/" /> | ||
| /// 3. Run the LocalAI examples. | ||
| /// </summary> | ||
| public class Example88_CustomMessageAPIEndpoint : BaseTest | ||
| { | ||
| [Theory(Skip = "Manual configuration needed")] | ||
| [InlineData("LMStudio", "http://localhost:1234", "llama2")] // Setup Llama2 as the model in LM Studio UI and start the Message API Server on http://localhost:1234 | ||
| [InlineData("Ollama", "http://localhost:11434", "llama2")] // Start the Ollama Message API Server on http://localhost:11434 using docker | ||
| [InlineData("LocalAI", "http://localhost:8080", "phi-2")] | ||
| public async Task LocalModel_ExampleAsync(string messageAPIPlatform, string url, string modelId) | ||
| { | ||
| WriteLine($"Example using local {messageAPIPlatform}"); | ||
| // Setup Llama2 as the model in LM Studio UI. | ||
|
|
||
| var kernel = Kernel.CreateBuilder() | ||
| .AddOpenAIChatCompletion( | ||
| modelId: modelId, | ||
| apiKey: null, | ||
| endpoint: new Uri(url)) | ||
| .Build(); | ||
|
|
||
| var prompt = @"Rewrite the text between triple backticks into a business mail. Use a professional tone, be clear and concise. | ||
| Sign the mail as AI Assistant. | ||
| Text: ```{{$input}}```"; | ||
|
|
||
| var mailFunction = kernel.CreateFunctionFromPrompt(prompt, new OpenAIPromptExecutionSettings | ||
| { | ||
| TopP = 0.5, | ||
| MaxTokens = 1000, | ||
| }); | ||
|
|
||
| var response = await kernel.InvokeAsync(mailFunction, new() { ["input"] = "Tell David that I'm going to finish the business plan by the end of the week." }); | ||
| this.WriteLine(response); | ||
| } | ||
|
|
||
| [Theory(Skip = "Manual configuration needed")] | ||
| [InlineData("LMStudio", "http://localhost:1234", "llama2")] // Setup Llama2 as the model in LM Studio UI and start the Message API Server on http://localhost:1234 | ||
| [InlineData("Ollama", "http://localhost:11434", "llama2")] // Start the Ollama Message API Server on http://localhost:11434 using docker | ||
| [InlineData("LocalAI", "http://localhost:8080", "phi-2")] | ||
| public async Task LocalModel_StreamingExampleAsync(string messageAPIPlatform, string url, string modelId) | ||
| { | ||
| WriteLine($"Example using local {messageAPIPlatform}"); | ||
|
|
||
| var kernel = Kernel.CreateBuilder() | ||
| .AddOpenAIChatCompletion( | ||
| modelId: modelId, | ||
| apiKey: null, | ||
| endpoint: new Uri(url)) | ||
| .Build(); | ||
|
|
||
| var prompt = @"Rewrite the text between triple backticks into a business mail. Use a professional tone, be clear and concise. | ||
| Sign the mail as AI Assistant. | ||
| Text: ```{{$input}}```"; | ||
|
|
||
| var mailFunction = kernel.CreateFunctionFromPrompt(prompt, new OpenAIPromptExecutionSettings | ||
| { | ||
| TopP = 0.5, | ||
| MaxTokens = 1000, | ||
| }); | ||
|
|
||
| await foreach (var word in kernel.InvokeStreamingAsync(mailFunction, new() { ["input"] = "Tell David that I'm going to finish the business plan by the end of the week." })) | ||
| { | ||
| this.WriteLine(word); | ||
| }; | ||
| } | ||
|
|
||
| public Example88_CustomMessageAPIEndpoint(ITestOutputHelper output) : base(output) | ||
| { | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
27 changes: 27 additions & 0 deletions
27
dotnet/src/Connectors/Connectors.OpenAI/AzureSdk/CustomHostPipelinePolicy.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System; | ||
| using Azure.Core; | ||
| using Azure.Core.Pipeline; | ||
|
|
||
| namespace Microsoft.SemanticKernel.Connectors.OpenAI.Core.AzureSdk; | ||
|
|
||
| internal class CustomHostPipelinePolicy : HttpPipelineSynchronousPolicy | ||
| { | ||
| private readonly Uri _endpoint; | ||
|
|
||
| internal CustomHostPipelinePolicy(Uri endpoint) | ||
| { | ||
| this._endpoint = endpoint; | ||
| } | ||
| public override void OnSendingRequest(HttpMessage message) | ||
| { | ||
| if (message?.Request == null) | ||
| { | ||
| return; | ||
| } | ||
|
|
||
| // Update current host to provided endpoint | ||
| message.Request.Uri.Reset(this._endpoint); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.