diff --git a/dotnet/samples/Demos/TelemetryWithAppInsights/Program.cs b/dotnet/samples/Demos/TelemetryWithAppInsights/Program.cs index 7fc1093c4d9d..dc1009bb74b3 100644 --- a/dotnet/samples/Demos/TelemetryWithAppInsights/Program.cs +++ b/dotnet/samples/Demos/TelemetryWithAppInsights/Program.cs @@ -77,11 +77,24 @@ public static async Task Main() Console.WriteLine(); Console.WriteLine("Write a poem about John Doe and translate it to Italian."); - await RunAzureOpenAIChatAsync(kernel); + using (var _ = s_activitySource.StartActivity("Chat")) + { + await RunAzureOpenAIChatAsync(kernel); + Console.WriteLine(); + await RunGoogleAIChatAsync(kernel); + Console.WriteLine(); + await RunHuggingFaceChatAsync(kernel); + } + Console.WriteLine(); - await RunGoogleAIChatAsync(kernel); Console.WriteLine(); - await RunHuggingFaceChatAsync(kernel); + + Console.WriteLine("Get weather."); + using (var _ = s_activitySource.StartActivity("ToolCalls")) + { + await RunAzureOpenAIToolCallsAsync(kernel); + Console.WriteLine(); + } } #region Private @@ -99,16 +112,17 @@ public static async Task Main() /// private static readonly ActivitySource s_activitySource = new("Telemetry.Example"); - private const string AzureOpenAIChatServiceKey = "AzureOpenAIChat"; - private const string GoogleAIGeminiChatServiceKey = "GoogleAIGeminiChat"; - private const string HuggingFaceChatServiceKey = "HuggingFaceChat"; + private const string AzureOpenAIServiceKey = "AzureOpenAI"; + private const string GoogleAIGeminiServiceKey = "GoogleAIGemini"; + private const string HuggingFaceServiceKey = "HuggingFace"; + #region chat completion private static async Task RunAzureOpenAIChatAsync(Kernel kernel) { Console.WriteLine("============= Azure OpenAI Chat Completion ============="); - using var activity = s_activitySource.StartActivity(AzureOpenAIChatServiceKey); - SetTargetService(kernel, AzureOpenAIChatServiceKey); + using var activity = s_activitySource.StartActivity(AzureOpenAIServiceKey); + SetTargetService(kernel, AzureOpenAIServiceKey); try { await RunChatAsync(kernel); @@ -124,8 +138,8 @@ private static async Task RunGoogleAIChatAsync(Kernel kernel) { Console.WriteLine("============= Google Gemini Chat Completion ============="); - using var activity = s_activitySource.StartActivity(GoogleAIGeminiChatServiceKey); - SetTargetService(kernel, GoogleAIGeminiChatServiceKey); + using var activity = s_activitySource.StartActivity(GoogleAIGeminiServiceKey); + SetTargetService(kernel, GoogleAIGeminiServiceKey); try { @@ -142,8 +156,8 @@ private static async Task RunHuggingFaceChatAsync(Kernel kernel) { Console.WriteLine("============= HuggingFace Chat Completion ============="); - using var activity = s_activitySource.StartActivity(HuggingFaceChatServiceKey); - SetTargetService(kernel, HuggingFaceChatServiceKey); + using var activity = s_activitySource.StartActivity(HuggingFaceServiceKey); + SetTargetService(kernel, HuggingFaceServiceKey); try { @@ -158,21 +172,54 @@ private static async Task RunHuggingFaceChatAsync(Kernel kernel) private static async Task RunChatAsync(Kernel kernel) { + // Using non-streaming to get the poem. var poem = await kernel.InvokeAsync( "WriterPlugin", "ShortPoem", new KernelArguments { ["input"] = "Write a poem about John Doe." }); - var translatedPoem = await kernel.InvokeAsync( + Console.WriteLine($"Poem:\n{poem}\n"); + + // Use streaming to translate the poem. + Console.WriteLine("Translated Poem:"); + await foreach (var update in kernel.InvokeStreamingAsync( "WriterPlugin", "Translate", new KernelArguments { ["input"] = poem, ["language"] = "Italian" - }); + })) + { + Console.Write(update); + } + } + #endregion + + #region tool calls + private static async Task RunAzureOpenAIToolCallsAsync(Kernel kernel) + { + Console.WriteLine("============= Azure OpenAI ToolCalls ============="); + + using var activity = s_activitySource.StartActivity(AzureOpenAIServiceKey); + SetTargetService(kernel, AzureOpenAIServiceKey); + try + { + await RunAutoToolCallAsync(kernel); + } + catch (Exception ex) + { + activity?.SetStatus(ActivityStatusCode.Error, ex.Message); + Console.WriteLine($"Error: {ex.Message}"); + } + } - Console.WriteLine($"Poem:\n{poem}\n\nTranslated Poem:\n{translatedPoem}"); + private static async Task RunAutoToolCallAsync(Kernel kernel) + { + var result = await kernel.InvokePromptAsync("What is the weather like in my location?"); + + Console.WriteLine(result); } + #endregion private static Kernel GetKernel(ILoggerFactory loggerFactory) { @@ -187,19 +234,21 @@ private static Kernel GetKernel(ILoggerFactory loggerFactory) modelId: TestConfiguration.AzureOpenAI.ChatModelId, endpoint: TestConfiguration.AzureOpenAI.Endpoint, apiKey: TestConfiguration.AzureOpenAI.ApiKey, - serviceId: AzureOpenAIChatServiceKey) + serviceId: AzureOpenAIServiceKey) .AddGoogleAIGeminiChatCompletion( modelId: TestConfiguration.GoogleAI.Gemini.ModelId, apiKey: TestConfiguration.GoogleAI.ApiKey, - serviceId: GoogleAIGeminiChatServiceKey) + serviceId: GoogleAIGeminiServiceKey) .AddHuggingFaceChatCompletion( model: TestConfiguration.HuggingFace.ModelId, endpoint: new Uri("https://api-inference.huggingface.co"), apiKey: TestConfiguration.HuggingFace.ApiKey, - serviceId: HuggingFaceChatServiceKey); + serviceId: HuggingFaceServiceKey); builder.Services.AddSingleton(new AIServiceSelector()); builder.Plugins.AddFromPromptDirectory(Path.Combine(folder, "WriterPlugin")); + builder.Plugins.AddFromType(); + builder.Plugins.AddFromType(); return builder.Build(); } @@ -240,9 +289,13 @@ private sealed class AIServiceSelector : IAIServiceSelector service = targetService; serviceSettings = targetServiceKey switch { - AzureOpenAIChatServiceKey => new OpenAIPromptExecutionSettings(), - GoogleAIGeminiChatServiceKey => new GeminiPromptExecutionSettings(), - HuggingFaceChatServiceKey => new HuggingFacePromptExecutionSettings(), + AzureOpenAIServiceKey => new OpenAIPromptExecutionSettings() + { + Temperature = 0, + ToolCallBehavior = ToolCallBehavior.AutoInvokeKernelFunctions + }, + GoogleAIGeminiServiceKey => new GeminiPromptExecutionSettings(), + HuggingFaceServiceKey => new HuggingFacePromptExecutionSettings(), _ => null, }; @@ -256,4 +309,23 @@ private sealed class AIServiceSelector : IAIServiceSelector } } #endregion + + #region Plugins + + public sealed class WeatherPlugin + { + [KernelFunction] + public string GetWeather(string location) => $"Weather in {location} is 70°F."; + } + + public sealed class LocationPlugin + { + [KernelFunction] + public string GetCurrentLocation() + { + return "Seattle"; + } + } + + #endregion } diff --git a/dotnet/samples/Demos/TelemetryWithAppInsights/TelemetryWithAppInsights.csproj b/dotnet/samples/Demos/TelemetryWithAppInsights/TelemetryWithAppInsights.csproj index 713b4043f3f3..26775e3a2402 100644 --- a/dotnet/samples/Demos/TelemetryWithAppInsights/TelemetryWithAppInsights.csproj +++ b/dotnet/samples/Demos/TelemetryWithAppInsights/TelemetryWithAppInsights.csproj @@ -7,7 +7,7 @@ disable false - $(NoWarn);CA1050;CA1707;CA2007;CS1591;VSTHRD111,SKEXP0050,SKEXP0060,SKEXP0070 + $(NoWarn);CA1024;CA1050;CA1707;CA2007;CS1591;VSTHRD111,SKEXP0050,SKEXP0060,SKEXP0070 5ee045b0-aea3-4f08-8d31-32d1a6f8fed0 diff --git a/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Clients/GeminiChatCompletionClient.cs b/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Clients/GeminiChatCompletionClient.cs index 8e19ddb09144..79b9089da5cb 100644 --- a/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Clients/GeminiChatCompletionClient.cs +++ b/dotnet/src/Connectors/Connectors.Google/Core/Gemini/Clients/GeminiChatCompletionClient.cs @@ -226,15 +226,56 @@ internal sealed class GeminiChatCompletionClient : ClientBase for (state.Iteration = 1; ; state.Iteration++) { - using var httpRequestMessage = await this.CreateHttpRequestAsync(state.GeminiRequest, this._chatStreamingEndpoint).ConfigureAwait(false); - using var response = await this.SendRequestAndGetResponseImmediatelyAfterHeadersReadAsync(httpRequestMessage, cancellationToken) - .ConfigureAwait(false); - using var responseStream = await response.Content.ReadAsStreamAndTranslateExceptionAsync() - .ConfigureAwait(false); - - await foreach (var messageContent in this.GetStreamingChatMessageContentsOrPopulateStateForToolCallingAsync(state, responseStream, cancellationToken).ConfigureAwait(false)) + using (var activity = ModelDiagnostics.StartCompletionActivity( + this._chatGenerationEndpoint, this._modelId, ModelProvider, chatHistory, executionSettings)) { - yield return messageContent; + HttpResponseMessage? httpResponseMessage = null; + Stream? responseStream = null; + try + { + using var httpRequestMessage = await this.CreateHttpRequestAsync(state.GeminiRequest, this._chatStreamingEndpoint).ConfigureAwait(false); + httpResponseMessage = await this.SendRequestAndGetResponseImmediatelyAfterHeadersReadAsync(httpRequestMessage, cancellationToken).ConfigureAwait(false); + responseStream = await httpResponseMessage.Content.ReadAsStreamAndTranslateExceptionAsync().ConfigureAwait(false); + } + catch (Exception ex) + { + activity?.SetError(ex); + httpResponseMessage?.Dispose(); + responseStream?.Dispose(); + throw; + } + + var responseEnumerator = this.GetStreamingChatMessageContentsOrPopulateStateForToolCallingAsync(state, responseStream, cancellationToken) + .GetAsyncEnumerator(cancellationToken); + List? streamedContents = activity is not null ? [] : null; + try + { + while (true) + { + try + { + if (!await responseEnumerator.MoveNextAsync().ConfigureAwait(false)) + { + break; + } + } + catch (Exception ex) + { + activity?.SetError(ex); + throw; + } + + streamedContents?.Add(responseEnumerator.Current); + yield return responseEnumerator.Current; + } + } + finally + { + activity?.EndStreaming(streamedContents); + httpResponseMessage?.Dispose(); + responseStream?.Dispose(); + await responseEnumerator.DisposeAsync().ConfigureAwait(false); + } } if (!state.AutoInvoke) diff --git a/dotnet/src/Connectors/Connectors.HuggingFace/Core/HuggingFaceClient.cs b/dotnet/src/Connectors/Connectors.HuggingFace/Core/HuggingFaceClient.cs index f93903094fad..a6c095738f1b 100644 --- a/dotnet/src/Connectors/Connectors.HuggingFace/Core/HuggingFaceClient.cs +++ b/dotnet/src/Connectors/Connectors.HuggingFace/Core/HuggingFaceClient.cs @@ -169,17 +169,53 @@ internal HttpRequestMessage CreatePost(object requestData, Uri endpoint, string? var request = this.CreateTextRequest(prompt, executionSettings); request.Stream = true; - using var httpRequestMessage = this.CreatePost(request, endpoint, this.ApiKey); - - using var response = await this.SendRequestAndGetResponseImmediatelyAfterHeadersReadAsync(httpRequestMessage, cancellationToken) - .ConfigureAwait(false); - - using var responseStream = await response.Content.ReadAsStreamAndTranslateExceptionAsync() - .ConfigureAwait(false); + using var activity = ModelDiagnostics.StartCompletionActivity(endpoint, modelId, this.ModelProvider, prompt, executionSettings); + HttpResponseMessage? httpResponseMessage = null; + Stream? responseStream = null; + try + { + using var httpRequestMessage = this.CreatePost(request, endpoint, this.ApiKey); + httpResponseMessage = await this.SendRequestAndGetResponseImmediatelyAfterHeadersReadAsync(httpRequestMessage, cancellationToken).ConfigureAwait(false); + responseStream = await httpResponseMessage.Content.ReadAsStreamAndTranslateExceptionAsync().ConfigureAwait(false); + } + catch (Exception ex) + { + activity?.SetError(ex); + httpResponseMessage?.Dispose(); + responseStream?.Dispose(); + throw; + } - await foreach (var streamingTextContent in this.ProcessTextResponseStreamAsync(responseStream, modelId, cancellationToken).ConfigureAwait(false)) + var responseEnumerator = this.ProcessTextResponseStreamAsync(responseStream, modelId, cancellationToken) + .GetAsyncEnumerator(cancellationToken); + List? streamedContents = activity is not null ? [] : null; + try + { + while (true) + { + try + { + if (!await responseEnumerator.MoveNextAsync().ConfigureAwait(false)) + { + break; + } + } + catch (Exception ex) + { + activity?.SetError(ex); + throw; + } + + streamedContents?.Add(responseEnumerator.Current); + yield return responseEnumerator.Current; + } + } + finally { - yield return streamingTextContent; + activity?.EndStreaming(streamedContents); + httpResponseMessage?.Dispose(); + responseStream?.Dispose(); + await responseEnumerator.DisposeAsync().ConfigureAwait(false); } } diff --git a/dotnet/src/Connectors/Connectors.HuggingFace/Core/HuggingFaceMessageApiClient.cs b/dotnet/src/Connectors/Connectors.HuggingFace/Core/HuggingFaceMessageApiClient.cs index 10b587788719..7ae142fb9cdd 100644 --- a/dotnet/src/Connectors/Connectors.HuggingFace/Core/HuggingFaceMessageApiClient.cs +++ b/dotnet/src/Connectors/Connectors.HuggingFace/Core/HuggingFaceMessageApiClient.cs @@ -85,17 +85,53 @@ internal sealed class HuggingFaceMessageApiClient var request = this.CreateChatRequest(chatHistory, executionSettings); request.Stream = true; - using var httpRequestMessage = this._clientCore.CreatePost(request, endpoint, this._clientCore.ApiKey); - - using var response = await this._clientCore.SendRequestAndGetResponseImmediatelyAfterHeadersReadAsync(httpRequestMessage, cancellationToken) - .ConfigureAwait(false); - - using var responseStream = await response.Content.ReadAsStreamAndTranslateExceptionAsync() - .ConfigureAwait(false); + using var activity = ModelDiagnostics.StartCompletionActivity(endpoint, modelId, this._clientCore.ModelProvider, chatHistory, executionSettings); + HttpResponseMessage? httpResponseMessage = null; + Stream? responseStream = null; + try + { + using var httpRequestMessage = this._clientCore.CreatePost(request, endpoint, this._clientCore.ApiKey); + httpResponseMessage = await this._clientCore.SendRequestAndGetResponseImmediatelyAfterHeadersReadAsync(httpRequestMessage, cancellationToken).ConfigureAwait(false); + responseStream = await httpResponseMessage.Content.ReadAsStreamAndTranslateExceptionAsync().ConfigureAwait(false); + } + catch (Exception ex) + { + activity?.SetError(ex); + httpResponseMessage?.Dispose(); + responseStream?.Dispose(); + throw; + } - await foreach (var streamingChatContent in this.ProcessChatResponseStreamAsync(responseStream, modelId, cancellationToken).ConfigureAwait(false)) + var responseEnumerator = this.ProcessChatResponseStreamAsync(responseStream, modelId, cancellationToken) + .GetAsyncEnumerator(cancellationToken); + List? streamedContents = activity is not null ? [] : null; + try + { + while (true) + { + try + { + if (!await responseEnumerator.MoveNextAsync().ConfigureAwait(false)) + { + break; + } + } + catch (Exception ex) + { + activity?.SetError(ex); + throw; + } + + streamedContents?.Add(responseEnumerator.Current); + yield return responseEnumerator.Current; + } + } + finally { - yield return streamingChatContent; + activity?.EndStreaming(streamedContents); + httpResponseMessage?.Dispose(); + responseStream?.Dispose(); + await responseEnumerator.DisposeAsync().ConfigureAwait(false); } } diff --git a/dotnet/src/Connectors/Connectors.OpenAI/AzureSdk/ClientCore.cs b/dotnet/src/Connectors/Connectors.OpenAI/AzureSdk/ClientCore.cs index aa2bb962ae6e..fac60f53903e 100644 --- a/dotnet/src/Connectors/Connectors.OpenAI/AzureSdk/ClientCore.cs +++ b/dotnet/src/Connectors/Connectors.OpenAI/AzureSdk/ClientCore.cs @@ -119,13 +119,13 @@ internal ClientCore(ILogger? logger = null) /// /// Creates completions for the prompt and settings. /// - /// The prompt to complete. + /// The prompt to complete. /// Execution settings for the completion API. /// The containing services, plugins, and other state for use throughout the operation. /// The to monitor for cancellation requests. The default is . /// Completions generated by the remote model internal async Task> GetTextResultsAsync( - string text, + string prompt, PromptExecutionSettings? executionSettings, Kernel? kernel, CancellationToken cancellationToken = default) @@ -134,11 +134,11 @@ internal ClientCore(ILogger? logger = null) ValidateMaxTokens(textExecutionSettings.MaxTokens); - var options = CreateCompletionsOptions(text, textExecutionSettings, this.DeploymentOrModelName); + var options = CreateCompletionsOptions(prompt, textExecutionSettings, this.DeploymentOrModelName); Completions? responseData = null; List responseContent; - using (var activity = ModelDiagnostics.StartCompletionActivity(this.Endpoint, this.DeploymentOrModelName, ModelProvider, text, executionSettings)) + using (var activity = ModelDiagnostics.StartCompletionActivity(this.Endpoint, this.DeploymentOrModelName, ModelProvider, prompt, executionSettings)) { try { @@ -183,15 +183,53 @@ internal ClientCore(ILogger? logger = null) var options = CreateCompletionsOptions(prompt, textExecutionSettings, this.DeploymentOrModelName); - StreamingResponse? response = await RunRequestAsync(() => this.Client.GetCompletionsStreamingAsync(options, cancellationToken)).ConfigureAwait(false); + using var activity = ModelDiagnostics.StartCompletionActivity(this.Endpoint, this.DeploymentOrModelName, ModelProvider, prompt, executionSettings); + + StreamingResponse response; + try + { + response = await RunRequestAsync(() => this.Client.GetCompletionsStreamingAsync(options, cancellationToken)).ConfigureAwait(false); + } + catch (Exception ex) + { + activity?.SetError(ex); + throw; + } - await foreach (Completions completions in response.ConfigureAwait(false)) + var responseEnumerator = response.ConfigureAwait(false).GetAsyncEnumerator(); + List? streamedContents = activity is not null ? [] : null; + try { - foreach (Choice choice in completions.Choices) + while (true) { - yield return new OpenAIStreamingTextContent(choice.Text, choice.Index, this.DeploymentOrModelName, choice, GetTextChoiceMetadata(completions, choice)); + try + { + if (!await responseEnumerator.MoveNextAsync()) + { + break; + } + } + catch (Exception ex) + { + activity?.SetError(ex); + throw; + } + + Completions completions = responseEnumerator.Current; + foreach (Choice choice in completions.Choices) + { + var openAIStreamingTextContent = new OpenAIStreamingTextContent( + choice.Text, choice.Index, this.DeploymentOrModelName, choice, GetTextChoiceMetadata(completions, choice)); + streamedContents?.Add(openAIStreamingTextContent); + yield return openAIStreamingTextContent; + } } } + finally + { + activity?.EndStreaming(streamedContents); + await responseEnumerator.DisposeAsync(); + } } private static Dictionary GetTextChoiceMetadata(Completions completions, Choice choice) @@ -613,9 +651,6 @@ static void AddResponseMessage(ChatCompletionsOptions chatOptions, ChatHistory c for (int requestIndex = 1; ; requestIndex++) { - // Make the request. - var response = await RunRequestAsync(() => this.Client.GetChatCompletionsStreamingAsync(chatOptions, cancellationToken)).ConfigureAwait(false); - // Reset state contentBuilder?.Clear(); toolCallIdsByIndex?.Clear(); @@ -627,25 +662,67 @@ static void AddResponseMessage(ChatCompletionsOptions chatOptions, ChatHistory c string? streamedName = null; ChatRole? streamedRole = default; CompletionsFinishReason finishReason = default; - await foreach (StreamingChatCompletionsUpdate update in response.ConfigureAwait(false)) + + using (var activity = ModelDiagnostics.StartCompletionActivity(this.Endpoint, this.DeploymentOrModelName, ModelProvider, chat, executionSettings)) { - metadata = GetResponseMetadata(update); - streamedRole ??= update.Role; - streamedName ??= update.AuthorName; - finishReason = update.FinishReason ?? default; + // Make the request. + StreamingResponse response; + try + { + response = await RunRequestAsync(() => this.Client.GetChatCompletionsStreamingAsync(chatOptions, cancellationToken)).ConfigureAwait(false); + } + catch (Exception ex) + { + activity?.SetError(ex); + throw; + } - // If we're intending to invoke function calls, we need to consume that function call information. - if (autoInvoke) + var responseEnumerator = response.ConfigureAwait(false).GetAsyncEnumerator(); + List? streamedContents = activity is not null ? [] : null; + try { - if (update.ContentUpdate is { Length: > 0 } contentUpdate) + while (true) { - (contentBuilder ??= new()).Append(contentUpdate); - } + try + { + if (!await responseEnumerator.MoveNextAsync()) + { + break; + } + } + catch (Exception ex) + { + activity?.SetError(ex); + throw; + } - OpenAIFunctionToolCall.TrackStreamingToolingUpdate(update.ToolCallUpdate, ref toolCallIdsByIndex, ref functionNamesByIndex, ref functionArgumentBuildersByIndex); - } + StreamingChatCompletionsUpdate update = responseEnumerator.Current; + metadata = GetResponseMetadata(update); + streamedRole ??= update.Role; + streamedName ??= update.AuthorName; + finishReason = update.FinishReason ?? default; - yield return new OpenAIStreamingChatMessageContent(update, update.ChoiceIndex ?? 0, this.DeploymentOrModelName, metadata) { AuthorName = streamedName }; + // If we're intending to invoke function calls, we need to consume that function call information. + if (autoInvoke) + { + if (update.ContentUpdate is { Length: > 0 } contentUpdate) + { + (contentBuilder ??= new()).Append(contentUpdate); + } + + OpenAIFunctionToolCall.TrackStreamingToolingUpdate(update.ToolCallUpdate, ref toolCallIdsByIndex, ref functionNamesByIndex, ref functionArgumentBuildersByIndex); + } + + var openAIStreamingChatMessageContent = new OpenAIStreamingChatMessageContent(update, update.ChoiceIndex ?? 0, this.DeploymentOrModelName, metadata) { AuthorName = streamedName }; + streamedContents?.Add(openAIStreamingChatMessageContent); + yield return openAIStreamingChatMessageContent; + } + } + finally + { + activity?.EndStreaming(streamedContents); + await responseEnumerator.DisposeAsync(); + } } // If we don't have a function to invoke, we're done. diff --git a/dotnet/src/InternalUtilities/src/Diagnostics/ModelDiagnostics.cs b/dotnet/src/InternalUtilities/src/Diagnostics/ModelDiagnostics.cs index 6ae98bb6e8e6..5522e0f73330 100644 --- a/dotnet/src/InternalUtilities/src/Diagnostics/ModelDiagnostics.cs +++ b/dotnet/src/InternalUtilities/src/Diagnostics/ModelDiagnostics.cs @@ -63,6 +63,18 @@ public static void SetCompletionResponse(this Activity activity, IEnumerable completions, int? promptTokens = null, int? completionTokens = null) => SetCompletionResponse(activity, completions, promptTokens, completionTokens, ToOpenAIFormat); + /// + /// Notify the end of streaming for a given activity. + /// + public static void EndStreaming(this Activity activity, IEnumerable? contents, int? promptTokens = null, int? completionTokens = null) + { + if (IsModelDiagnosticsEnabled()) + { + var choices = OrganizeStreamingContent(contents); + SetCompletionResponse(activity, choices, promptTokens, completionTokens); + } + } + /// /// Set the response id for a given activity. /// @@ -87,16 +99,16 @@ public static void SetCompletionResponse(this Activity activity, IEnumerableThe activity with the completion token usage set for chaining public static Activity SetCompletionTokenUsage(this Activity activity, int completionTokens) => activity.SetTag(ModelDiagnosticsTags.CompletionToken, completionTokens); - # region Private /// /// Check if model diagnostics is enabled /// Model diagnostics is enabled if either EnableModelDiagnostics or EnableSensitiveEvents is set to true and there are listeners. /// - private static bool IsModelDiagnosticsEnabled() + public static bool IsModelDiagnosticsEnabled() { return (s_enableDiagnostics || s_enableSensitiveEvents) && s_activitySource.HasListeners(); } + #region Private private static void AddOptionalTags(Activity? activity, PromptExecutionSettings? executionSettings) { if (activity is null || executionSettings?.ExtensionData is null) @@ -136,9 +148,11 @@ private static string ToOpenAIFormat(IEnumerable chatHistory sb.Append("{\"role\": \""); sb.Append(message.Role); - sb.Append("\", \"content\": \""); + sb.Append("\", \"content\": "); sb.Append(JsonSerializer.Serialize(message.Content)); - sb.Append("\"}"); + sb.Append(", \"tool_calls\": "); + ToOpenAIFormat(sb, message.Items); + sb.Append('}'); isFirst = false; } @@ -147,6 +161,35 @@ private static string ToOpenAIFormat(IEnumerable chatHistory return sb.ToString(); } + /// + /// Helper method to convert tool calls to a string aligned with the OpenAI format + /// + private static void ToOpenAIFormat(StringBuilder sb, ChatMessageContentItemCollection chatMessageContentItems) + { + sb.Append('['); + var isFirst = true; + foreach (var functionCall in chatMessageContentItems.OfType()) + { + if (!isFirst) + { + // Append a comma and a newline to separate the elements after the previous one. + // This can avoid adding an unnecessary comma after the last element. + sb.Append(", \n"); + } + + sb.Append("{\"id\": \""); + sb.Append(functionCall.Id); + sb.Append("\", \"function\": {\"arguments\": "); + sb.Append(JsonSerializer.Serialize(functionCall.Arguments)); + sb.Append(", \"name\": \""); + sb.Append(functionCall.FunctionName); + sb.Append("\"}, \"type\": \"function\"}"); + + isFirst = false; + } + sb.Append(']'); + } + /// /// Start a completion activity and return the activity. /// The `formatPrompt` delegate won't be invoked if events are disabled. @@ -238,6 +281,44 @@ private static string ToOpenAIFormat(IEnumerable chatHistory } } + /// + /// Set the streaming completion response for a given activity. + /// + private static void SetCompletionResponse( + Activity activity, + Dictionary> choices, + int? promptTokens, + int? completionTokens) + { + if (!IsModelDiagnosticsEnabled()) + { + return; + } + + // Assuming all metadata is in the last chunk of the choice + switch (choices.FirstOrDefault().Value.FirstOrDefault()) + { + case StreamingTextContent: + var textCompletions = choices.Select(choiceContents => + { + var lastContent = (StreamingTextContent)choiceContents.Value.Last(); + var text = choiceContents.Value.Select(c => c.ToString()).Aggregate((a, b) => a + b); + return new TextContent(text, metadata: lastContent.Metadata); + }).ToList(); + SetCompletionResponse(activity, textCompletions, promptTokens, completionTokens, completions => $"[{string.Join(", ", completions)}"); + break; + case StreamingChatMessageContent: + var chatCompletions = choices.Select(choiceContents => + { + var lastContent = (StreamingChatMessageContent)choiceContents.Value.Last(); + var chatMessage = choiceContents.Value.Select(c => c.ToString()).Aggregate((a, b) => a + b); + return new ChatMessageContent(lastContent.Role ?? AuthorRole.Assistant, chatMessage, metadata: lastContent.Metadata); + }).ToList(); + SetCompletionResponse(activity, chatCompletions, promptTokens, completionTokens, ToOpenAIFormat); + break; + }; + } + // Returns an activity for chaining private static Activity SetFinishReasons(this Activity activity, IEnumerable completions) { @@ -270,6 +351,31 @@ private static Activity SetResponseId(this Activity activity, KernelContent? com return activity; } + /// + /// Organize streaming content by choice index + /// + private static Dictionary> OrganizeStreamingContent(IEnumerable? contents) + { + Dictionary> choices = []; + if (contents is null) + { + return choices; + } + + foreach (var content in contents) + { + if (!choices.TryGetValue(content.ChoiceIndex, out var choiceContents)) + { + choiceContents = []; + choices[content.ChoiceIndex] = choiceContents; + } + + choiceContents.Add(content); + } + + return choices; + } + /// /// Tags used in model diagnostics ///