fix hardcoded model for chat with images, add exception for chat with audio files, update tests

Apps.OpenAI/Actions/ChatActions.cs

@@ -5,10 +5,10 @@
 using Apps.OpenAI.Models.Requests.Chat;
 using Apps.OpenAI.Models.Responses.Chat;
 using Apps.OpenAI.Utils;
+using Apps.OpenAI.Extensions;
 using Blackbird.Applications.Sdk.Common;
 using Blackbird.Applications.Sdk.Common.Actions;
 using Blackbird.Applications.Sdk.Common.Exceptions;
-using Blackbird.Applications.Sdk.Common.Files;
 using Blackbird.Applications.Sdk.Common.Invocation;
 using Blackbird.Applications.Sdk.Utils.Extensions.Files;
 using Blackbird.Applications.SDK.Extensions.FileManagement.Interfaces;
@@ -35,7 +35,7 @@ public async Task<ChatResponse> ChatMessageRequest(
         [ActionParameter] ChatRequest input,
         [ActionParameter] GlossaryRequest glossary)
     {
-        HandleInput(modelIdentifier, input);
+        HandleInput(input);
 
         var messages = await GenerateChatMessages(input, glossary);
         var completeMessage = string.Empty;
@@ -97,7 +97,7 @@ public async Task<ChatResponse> ChatWithSystemMessageRequest(
             Message = input.Message,
             MaximumTokens = input.MaximumTokens,
             FrequencyPenalty = input.FrequencyPenalty,
-            File = input.Image,
+            File = input.File,
             Parameters = input.Parameters,
             PresencePenalty = input.PresencePenalty,
             Temperature = input.Temperature,
@@ -127,22 +127,22 @@ private async Task<List<BaseChatMessageDto>> GenerateChatMessages(ChatRequest in
             var fileStream = await FileManagementClient.DownloadAsync(input.File);
             var fileBytes = await fileStream.GetByteData();
 
-            if (input.File.ContentType.StartsWith("audio") || input.File.Name.EndsWith("wav") || input.File.Name.EndsWith("mp3"))
+            if (input.File.IsAudio())
             {
-                messages.Add(new ChatAudioMessageDto(MessageRoles.User, new List<ChatAudioMessageContentDto>
-                {
-
-                    new ChatAudioMessageTextContentDto("text", input.Message),
-                    new ChatAudioMessageAudioContentDto("input_audio", new InputAudio(){Format = input.File.Name.Substring(input.File.Name.Length-3).ToLower(),Data = Convert.ToBase64String(fileBytes) })
-                }));
+                // The Completions API did support audio inputs, but the Responses API doesn't
+                throw new PluginMisconfigurationException(
+                    "OpenAI does not support audio files for chat endpoints. " +
+                    "Please use Audio actions for such files");
             }
-            else if (input.File.ContentType.StartsWith("image") || input.File.Name.EndsWith("png") || input.File.Name.EndsWith("jpg") || input.File.Name.EndsWith("jpeg") || input.File.Name.EndsWith("webp") || input.File.Name.EndsWith("gif"))
+
+            if (input.File.IsImage())
             {
                 messages.Add(new ChatImageMessageDto(MessageRoles.User, new List<ChatImageMessageContentDto>
                 {
                     new ChatImageMessageTextContentDto("text", input.Message),
-                    new ChatImageMessageImageContentDto("image_url", new ImageUrlDto(
-                        $"data:{input.File.ContentType};base64,{Convert.ToBase64String(fileBytes)}"))
+                    new ChatImageMessageImageContentDto(
+                        "image_url", 
+                        new ImageUrlDto($"data:{input.File.ContentType};base64,{Convert.ToBase64String(fileBytes)}"))
                 }));
             }
             else
@@ -199,59 +199,12 @@ private async Task<List<BaseChatMessageDto>> GenerateChatMessages(ChatRequest in
         return messages;
     }
 
-    private void HandleInput(TextChatModelIdentifier modelIdentifier, ChatRequest input)
+    private void HandleInput(ChatRequest input)
     {
         if (input.EnableWebSearch == true && UniversalClient.ConnectionType == ConnectionTypes.AzureOpenAi)
-        {
-            throw new PluginMisconfigurationException("Web search is not supported for Azure OpenAI connections in this action. Please use an OpenAI connection.");
-        }
-
-        if (UniversalClient.ConnectionType == ConnectionTypes.OpenAi)
-        {
-            if (string.IsNullOrEmpty(modelIdentifier.ModelId))
-                throw new PluginMisconfigurationException("Please select a model to execute this action using the OpenAI connection");
-            HandleOpenAiFileInput(modelIdentifier, input.File);
-        }
-        else if (UniversalClient.ConnectionType == ConnectionTypes.AzureOpenAi)
-            HandleAzureFileInput(input.File);
-        else HandleOpenAiFileInput(modelIdentifier, input.File);
-    }
-
-    private static void HandleOpenAiFileInput(TextChatModelIdentifier modelIdentifier, FileReference? file)
-    {
-        if (file == null) return;
-
-        var name = file.Name.ToLowerInvariant();
-        var type = file.ContentType.ToLowerInvariant();
-
-        if (IsAudioFile(name, type))
-        {
-            modelIdentifier.ModelId = "gpt-4o-audio-preview";
-        }
-        else if (IsImageFile(name, type))
-        {
-            modelIdentifier.ModelId = "gpt-4-vision-preview";
-        }
-    }
-
-    private static void HandleAzureFileInput(FileReference? file)
-    {
-        if (file == null) return;
-
-        var name = file.Name.ToLowerInvariant();
-        var type = file.ContentType.ToLowerInvariant();
-
-        if (IsAudioFile(name, type))
         {
             throw new PluginMisconfigurationException(
-                "Azure OpenAI does not support chat actions with audio files. Please use OpenAI for such tasks"
-            );
+                "Web search is not supported for Azure OpenAI connections in this action. Please use an OpenAI connection.");
         }
     }
-
-    private static bool IsAudioFile(string name, string contentType) =>
-        contentType.StartsWith("audio") || name.EndsWith(".wav") || name.EndsWith(".mp3");
-
-    private static bool IsImageFile(string name, string contentType) =>
-        contentType.StartsWith("image") || new[] { ".png", ".jpg", ".jpeg", ".webp", ".gif" }.Any(name.EndsWith);
 }

Apps.OpenAI/Apps.OpenAI.csproj

@@ -4,7 +4,7 @@
     <TargetFramework>net8.0</TargetFramework>
 	  <Product>OpenAI</Product>
 	  <Description>Creating safe artificial general intelligence that benefits all of humanity</Description>
-	  <Version>2.8.38</Version>
+	  <Version>2.8.39</Version>
 	  <AssemblyName>Apps.OpenAI</AssemblyName>
   </PropertyGroup>
 

Apps.OpenAI/Extensions/FileReferenceExtensions.cs

@@ -0,0 +1,22 @@
+using System.Linq;
+using Blackbird.Applications.Sdk.Common.Files;
+
+namespace Apps.OpenAI.Extensions;
+
+public static class FileReferenceExtensions
+{
+    public static bool IsAudio(this FileReference fileReference)
+    {
+        return 
+            fileReference.ContentType.StartsWith("audio") || 
+            fileReference.Name.EndsWith(".wav") || 
+            fileReference.Name.EndsWith(".mp3");
+    }
+
+    public static bool IsImage(this FileReference fileReference)
+    {
+        return
+            fileReference.ContentType.StartsWith("image") || 
+            new[] { ".png", ".jpg", ".jpeg", ".webp", ".gif" }.Any(fileReference.Name.EndsWith);
+    }
+}

Apps.OpenAI/Models/Requests/Chat/ChatRequestWithSystem.cs

@@ -18,7 +18,7 @@ public class ChatRequestWithSystem : BaseChatRequest, IWebSearchRequest
             "Texts that will be added to the user prompt along with the message. Useful if you want to add collection of messages to the prompt.")]
     public IEnumerable<string>? Parameters { get; set; }
 
-    public FileReference? Image { get; set; }
+    public FileReference? File { get; set; }
 
     [Display("Enable web search")]
     public bool? EnableWebSearch { get; set; }

Tests.OpenAI/ChatActionsTests.cs

@@ -78,8 +78,8 @@ public async Task ChatMessageRequest_WithHtmlFile_ReturnsValidResponse(Invocatio
         Assert.IsNotNull(result.Message);
     }
 
-    [TestMethod, ContextDataSource(ConnectionTypes.OpenAiEmbedded)]
-    public async Task ChatMessageRequest_OpenAiEmbeddedWithAudioFile_ReturnsValidResponse(InvocationContext context)
+    [TestMethod, ContextDataSource(ConnectionTypes.OpenAi)]
+    public async Task ChatMessageRequest_OpenAiWithAudioFile_ThrowsMisconfigException(InvocationContext context)
     {
         // Arrange
         var actions = new ChatActions(context, FileManagementClient);
@@ -96,11 +96,11 @@ public async Task ChatMessageRequest_OpenAiEmbeddedWithAudioFile_ReturnsValidRes
         var glossary = new GlossaryRequest();
 
         // Act
-        var result = await actions.ChatMessageRequest(modelIdentifier, chatRequest, glossary);
+        var ex = await Assert.ThrowsExactlyAsync<PluginMisconfigurationException>(() =>
+            actions.ChatMessageRequest(modelIdentifier, chatRequest, glossary));
 
         // Assert
-        PrintResult(result);
-        Assert.IsNotNull(result.Message);
+        Assert.Contains("use Audio actions", ex.Message);
     }
 
     [TestMethod, ContextDataSource(ConnectionTypes.OpenAi)]
@@ -252,4 +252,33 @@ public async Task ChatMessageRequest_Gpt5WithHighReasoningAndWebSearch_ReturnsVa
         PrintResult(result);
         Assert.IsNotNull(result.Message);
     }
+
+    [TestMethod, ContextDataSource(ConnectionTypes.OpenAi)]
+    public async Task ChatWithSystemMessageRequest_OpenAiWithImage_ReturnsValidResponse(InvocationContext context)
+    {
+        // Arrange
+        var action = new ChatActions(context, FileManagementClient);
+        var modelIdentifier = new TextChatModelIdentifier
+        {
+            ModelId = "gpt-5"
+        };
+        var input = new ChatRequestWithSystem
+        {
+            SystemPrompt = "You are very funny assistant",
+            Message = "Please describe this image",
+            File = new FileReference
+            {
+                Name = "tyrol-italy.jpg",
+                ContentType = "image/jpeg"
+            }
+        };
+        var glossary = new GlossaryRequest();
+
+        // Act
+        var result = await action.ChatWithSystemMessageRequest(modelIdentifier, input, glossary);
+
+        // Assert
+        PrintResult(result);
+        Assert.IsNotNull(result.Message);
+    }
 }