.Net: Support BinaryContent in OpenAI Connector (#11644)

### Motivation and Context

The OpenAI Connector only supports `TextContent`, `ImageContent`, and
`AudioContent` KernelContent types. However, the Chat Completion API
supports uploading other file types encoded as a base64 string. This
change leverages that behavior to add support for `BinaryContent`.

### Description

This change adds support for `BinaryContent` KernelContent types by
using `ChatMessageContentPart.CreateFilePart` to create the
corresponding `ChatMessageContentPart`.

### Contribution Checklist

- [x] The code builds clean without any errors or warnings
- [x] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [x] All unit tests pass, and I have added new tests where possible
- [x] I didn't break anyone 😄

---------

Co-authored-by: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com>

Author: nkeenan38

dotnet/samples/Concepts/ChatCompletion/OpenAI_ChatCompletionWithFile.cs

@@ -0,0 +1,69 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.ChatCompletion;
+using Resources;
+
+namespace ChatCompletion;
+
+/// <summary>
+/// This example shows how to use binary files input with OpenAI's chat completion.
+/// </summary>
+public class OpenAI_ChatCompletionWithFile(ITestOutputHelper output) : BaseTest(output)
+{
+    /// <summary>
+    /// This uses a local file as input for your chat
+    /// </summary>
+    [Fact]
+    public async Task UsingLocalFileInChatCompletion()
+    {
+        var fileBytes = await EmbeddedResource.ReadAllAsync("employees.pdf");
+
+        var kernel = Kernel.CreateBuilder()
+            .AddOpenAIChatCompletion(TestConfiguration.OpenAI.ChatModelId, TestConfiguration.OpenAI.ApiKey)
+            .Build();
+
+        var chatCompletionService = kernel.GetRequiredService<IChatCompletionService>();
+
+        var chatHistory = new ChatHistory("You are a friendly assistant.");
+
+        chatHistory.AddUserMessage(
+        [
+            new TextContent("What's in this file?"),
+            new BinaryContent(fileBytes, "application/pdf")
+        ]);
+
+        var reply = await chatCompletionService.GetChatMessageContentAsync(chatHistory);
+
+        Console.WriteLine(reply.Content);
+    }
+
+    /// <summary>
+    /// This uses a Base64 data URI as a binary file input for your chat
+    /// </summary>
+    [Fact]
+    public async Task UsingBase64DataUriInChatCompletion()
+    {
+        var fileBytes = await EmbeddedResource.ReadAllAsync("employees.pdf");
+        var fileBase64 = Convert.ToBase64String(fileBytes.ToArray());
+        var dataUri = $"data:application/pdf;base64,{fileBase64}";
+
+        var kernel = Kernel.CreateBuilder()
+            .AddOpenAIChatCompletion(TestConfiguration.OpenAI.ChatModelId, TestConfiguration.OpenAI.ApiKey)
+            .Build();
+
+        var chatCompletionService = kernel.GetRequiredService<IChatCompletionService>();
+
+        var chatHistory = new ChatHistory("You are a friendly assistant.");
+
+        chatHistory.AddUserMessage(
+        [
+            new TextContent("What's in this file?"),
+            new BinaryContent(dataUri)
+        ]);
+
+        var reply = await chatCompletionService.GetChatMessageContentAsync(chatHistory);
+
+        Console.WriteLine(reply.Content);
+    }
+}

dotnet/samples/Concepts/README.md

@@ -81,6 +81,7 @@ dotnet test -l "console;verbosity=detailed" --filter "FullyQualifiedName=ChatCom
 - [OpenAI_ChatCompletionStreaming](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/OpenAI_ChatCompletionStreaming.cs)
 - [OpenAI_ChatCompletionWebSearch](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/OpenAI_ChatCompletionWebSearch.cs)
 - [OpenAI_ChatCompletionWithAudio](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/OpenAI_ChatCompletionWithAudio.cs)
+- [OpenAI_ChatCompletionWithFile](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/OpenAI_ChatCompletionWithFile.cs)
 - [OpenAI_ChatCompletionWithReasoning](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/OpenAI_ChatCompletionWithReasoning.cs)
 - [OpenAI_ChatCompletionWithVision](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/OpenAI_ChatCompletionWithVision.cs)
 - [OpenAI_CustomClient](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/OpenAI_CustomClient.cs)

dotnet/samples/Concepts/Resources/employees.pdf

@@ -2144,4 +2144,85 @@ public async Task ItHandlesAudioContentWithMetadataInResponseAsync()
         Assert.NotNull(audioContent.Metadata["ExpiresAt"]);
         // The ExpiresAt value is converted to a DateTime object, so we can't directly compare it to the Unix timestamp
     }
+
+    [Fact]
+    public async Task GetChatMessageContentsThrowsExceptionWithEmptyBinaryContentAsync()
+    {
+        // Arrange
+        var chatCompletion = new OpenAIChatCompletionService(modelId: "gpt-4o-mini", apiKey: "NOKEY", httpClient: this._httpClient);
+
+        var chatHistory = new ChatHistory();
+        chatHistory.AddUserMessage([new Microsoft.SemanticKernel.BinaryContent()]);
+
+        // Act & Assert
+        await Assert.ThrowsAsync<ArgumentException>(() => chatCompletion.GetChatMessageContentsAsync(chatHistory));
+    }
+
+    [Fact]
+    public async Task GetChatMessageContentsThrowsExceptionUriOnlyReferenceBinaryContentAsync()
+    {
+        // Arrange
+        var chatCompletion = new OpenAIChatCompletionService(modelId: "gpt-4o-mini", apiKey: "NOKEY", httpClient: this._httpClient);
+
+        var chatHistory = new ChatHistory();
+        chatHistory.AddUserMessage([new Microsoft.SemanticKernel.BinaryContent(new Uri("file://testfile.pdf"))]);
+
+        // Act & Assert
+        await Assert.ThrowsAsync<ArgumentException>(() => chatCompletion.GetChatMessageContentsAsync(chatHistory));
+    }
+
+    [Theory]
+    [InlineData(true)]
+    [InlineData(false)]
+    public async Task ItSendsBinaryContentCorrectlyAsync(bool useUriData)
+    {
+        // Arrange
+        var chatCompletion = new OpenAIChatCompletionService(modelId: "gpt-4o-mini", apiKey: "NOKEY", httpClient: this._httpClient);
+        this._messageHandlerStub.ResponseToReturn = new HttpResponseMessage(System.Net.HttpStatusCode.OK)
+        {
+            Content = new StringContent(ChatCompletionResponse)
+        };
+
+        var mimeType = "application/pdf";
+        var chatHistory = new ChatHistory();
+        chatHistory.AddUserMessage([
+            new TextContent("What's in this file?"),
+            useUriData
+                ? new Microsoft.SemanticKernel.BinaryContent($"data:{mimeType};base64,{PdfBase64Data}")
+                : new Microsoft.SemanticKernel.BinaryContent(Convert.FromBase64String(PdfBase64Data), mimeType)
+        ]);
+
+        // Act
+        await chatCompletion.GetChatMessageContentsAsync(chatHistory);
+
+        // Assert
+        var actualRequestContent = Encoding.UTF8.GetString(this._messageHandlerStub.RequestContent!);
+        Assert.NotNull(actualRequestContent);
+        var optionsJson = JsonSerializer.Deserialize<JsonElement>(actualRequestContent);
+
+        var messages = optionsJson.GetProperty("messages");
+        Assert.Equal(1, messages.GetArrayLength());
+
+        var contentItems = messages[0].GetProperty("content");
+        Assert.Equal(2, contentItems.GetArrayLength());
+
+        Assert.Equal("text", contentItems[0].GetProperty("type").GetString());
+        Assert.Equal("What's in this file?", contentItems[0].GetProperty("text").GetString());
+
+        Assert.Equal("file", contentItems[1].GetProperty("type").GetString());
+
+        // Check for the file data
+        Assert.True(contentItems[1].TryGetProperty("file", out var fileData));
+        Assert.Equal(JsonValueKind.Object, fileData.ValueKind);
+        Assert.True(fileData.TryGetProperty("file_data", out var dataProperty));
+        var dataUriFile = dataProperty.GetString();
+
+        Assert.NotNull(dataUriFile);
+        Assert.Equal($"data:{mimeType};base64,{PdfBase64Data}", dataUriFile);
+    }
+
+    /// <summary>
+    /// Sample PDF data URI for testing.
+    /// </summary>
+    private const string PdfBase64Data = "JVBERi0xLjQKMSAwIG9iago8PC9UeXBlIC9DYXRhbG9nCi9QYWdlcyAyIDAgUgo+PgplbmRvYmoKMiAwIG9iago8PC9UeXBlIC9QYWdlcwovS2lkcyBbMyAwIFJdCi9Db3VudCAxCj4+CmVuZG9iagozIDAgb2JqCjw8L1R5cGUgL1BhZ2UKL1BhcmVudCAyIDAgUgovTWVkaWFCb3ggWzAgMCA1OTUgODQyXQovQ29udGVudHMgNSAwIFIKL1Jlc291cmNlcyA8PC9Qcm9jU2V0IFsvUERGIC9UZXh0XQovRm9udCA8PC9GMSA0IDAgUj4+Cj4+Cj4+CmVuZG9iago0IDAgb2JqCjw8L1R5cGUgL0ZvbnQKL1N1YnR5cGUgL1R5cGUxCi9OYW1lIC9GMQovQmFzZUZvbnQgL0hlbHZldGljYQovRW5jb2RpbmcgL01hY1JvbWFuRW5jb2RpbmcKPj4KZW5kb2JqCjUgMCBvYmoKPDwvTGVuZ3RoIDUzCj4+CnN0cmVhbQpCVAovRjEgMjAgVGYKMjIwIDQwMCBUZAooRHVtbXkgUERGKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZgowMDAwMDAwMDA5IDAwMDAwIG4KMDAwMDAwMDA2MyAwMDAwMCBuCjAwMDAwMDAxMjQgMDAwMDAgbgowMDAwMDAwMjc3IDAwMDAwIG4KMDAwMDAwMDM5MiAwMDAwMCBuCnRyYWlsZXIKPDwvU2l6ZSA2Ci9Sb290IDEgMCBSCj4+CnN0YXJ0eHJlZgo0OTUKJSVFT0YK";
 }

dotnet/src/Connectors/Connectors.OpenAI.UnitTests/Services/OpenAIChatCompletionServiceTests.cs

@@ -777,6 +777,7 @@ private static List<ChatMessage> CreateRequestMessages(ChatMessageContent messag
                         TextContent textContent => ChatMessageContentPart.CreateTextPart(textContent.Text),
                         ImageContent imageContent => GetImageContentItem(imageContent),
                         AudioContent audioContent => GetAudioContentItem(audioContent),
+                        BinaryContent binaryContent => GetBinaryContentItem(binaryContent),
                         _ => throw new NotSupportedException($"Unsupported chat message content type '{item.GetType()}'.")
                     }))
                 { ParticipantName = message.AuthorName }
@@ -887,6 +888,16 @@ private static ChatMessageContentPart GetAudioContentItem(AudioContent audioCont
         throw new ArgumentException($"{nameof(AudioContent)} must have Data bytes.");
     }
 
+    private static ChatMessageContentPart GetBinaryContentItem(BinaryContent binaryContent)
+    {
+        if (binaryContent.Data is { IsEmpty: false } data)
+        {
+            return ChatMessageContentPart.CreateFilePart(BinaryData.FromBytes(data), binaryContent.MimeType, Guid.NewGuid().ToString());
+        }
+
+        throw new ArgumentException($"{nameof(BinaryContent)} must have Data bytes.");
+    }
+
     private static ChatInputAudioFormat GetChatInputAudioFormat(string? mimeType)
     {
         if (string.IsNullOrWhiteSpace(mimeType))