diff --git a/DevProxy.Abstractions/LanguageModel/OpenAIModels.cs b/DevProxy.Abstractions/LanguageModel/OpenAIModels.cs index 3d2496b6..93742eca 100644 --- a/DevProxy.Abstractions/LanguageModel/OpenAIModels.cs +++ b/DevProxy.Abstractions/LanguageModel/OpenAIModels.cs @@ -42,6 +42,15 @@ public static bool TryGetOpenAIRequest(string content, ILogger logger, out OpenA var rawRequest = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); + // Responses API request (check first as it's the recommended API) + if (rawRequest.TryGetProperty("input", out _) && + rawRequest.TryGetProperty("modalities", out _)) + { + logger.LogDebug("Request is a Responses API request"); + request = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); + return true; + } + // Check for completion request (has "prompt", but not specific to image) if (rawRequest.TryGetProperty("prompt", out _) && !rawRequest.TryGetProperty("size", out _) && @@ -63,7 +72,8 @@ public static bool TryGetOpenAIRequest(string content, ILogger logger, out OpenA // Embedding request if (rawRequest.TryGetProperty("input", out _) && rawRequest.TryGetProperty("model", out _) && - !rawRequest.TryGetProperty("voice", out _)) + !rawRequest.TryGetProperty("voice", out _) && + !rawRequest.TryGetProperty("modalities", out _)) { logger.LogDebug("Request is an embedding request"); request = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); @@ -409,3 +419,69 @@ public class OpenAIImageData [JsonPropertyName("revised_prompt")] public string? RevisedPrompt { get; set; } } + +#region Responses API + +public class OpenAIResponsesRequest : OpenAIRequest +{ + public object? Input { get; set; } + public IEnumerable? Modalities { get; set; } + public string? Instructions { get; set; } + public bool? Store { get; set; } + [JsonPropertyName("previous_response_id")] + public string? PreviousResponseId { get; set; } + public object? Tools { get; set; } + [JsonPropertyName("max_output_tokens")] + public long? MaxOutputTokens { get; set; } +} + +public class OpenAIResponsesResponse : OpenAIResponse +{ + public IEnumerable? Output { get; set; } + [JsonPropertyName("created_at")] + public long CreatedAt { get; set; } + public string? Status { get; set; } + + public override string? Response + { + get + { + if (Output is null || !Output.Any()) + { + return null; + } + + // Find the last message-type output item with text content + var lastMessage = Output + .Where(item => item.Type == "message") + .LastOrDefault(); + + if (lastMessage?.Content is null) + { + return null; + } + + // Extract text from content array + var textContent = lastMessage.Content + .Where(c => c.Type == "output_text") + .LastOrDefault(); + + return textContent?.Text; + } + } +} + +public class OpenAIResponsesOutputItem +{ + public string? Type { get; set; } + public string? Role { get; set; } + public IEnumerable? Content { get; set; } +} + +public class OpenAIResponsesContentPart +{ + public string? Type { get; set; } + public string? Text { get; set; } +} + +#endregion diff --git a/DevProxy.Plugins/Behavior/LanguageModelFailurePlugin.cs b/DevProxy.Plugins/Behavior/LanguageModelFailurePlugin.cs index 9995208f..42e5e107 100644 --- a/DevProxy.Plugins/Behavior/LanguageModelFailurePlugin.cs +++ b/DevProxy.Plugins/Behavior/LanguageModelFailurePlugin.cs @@ -73,7 +73,7 @@ public override async Task BeforeRequestAsync(ProxyRequestArgs e, CancellationTo return; } - if (!TryGetOpenAIRequest(request.BodyString, out var openAiRequest)) + if (!OpenAIRequest.TryGetOpenAIRequest(request.BodyString, Logger, out var openAiRequest)) { Logger.LogRequest("Skipping non-OpenAI request", MessageType.Skipped, new(e.Session)); return; @@ -116,53 +116,52 @@ public override async Task BeforeRequestAsync(ProxyRequestArgs e, CancellationTo Logger.LogRequest($"Simulating fault {faultName}", MessageType.Chaos, new(e.Session)); e.Session.SetRequestBodyString(JsonSerializer.Serialize(newRequest, ProxyUtils.JsonSerializerOptions)); } - else - { - Logger.LogDebug("Unknown OpenAI request type. Passing request as-is."); - } - - await Task.CompletedTask; - - Logger.LogTrace("Left {Name}", nameof(BeforeRequestAsync)); - } - - private bool TryGetOpenAIRequest(string content, out OpenAIRequest? request) - { - request = null; - - if (string.IsNullOrEmpty(content)) - { - return false; - } - - try + else if (openAiRequest is OpenAIResponsesRequest responsesRequest) { - Logger.LogDebug("Checking if the request is an OpenAI request..."); - - var rawRequest = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); - - if (rawRequest.TryGetProperty("prompt", out _)) + // Handle Responses API + if (responsesRequest.Input is string inputString) { - Logger.LogDebug("Request is a completion request"); - request = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); - return true; + // Simple string input - append fault prompt + responsesRequest.Input = inputString + "\n\n" + faultPrompt; + Logger.LogDebug("Modified Responses API string input with fault prompt"); } - - if (rawRequest.TryGetProperty("messages", out _)) + else if (responsesRequest.Input is JsonElement inputElement) { - Logger.LogDebug("Request is a chat completion request"); - request = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); - return true; + // Structured input - append as new message item + try + { + var items = JsonSerializer.Deserialize>(inputElement.GetRawText(), ProxyUtils.JsonSerializerOptions) ?? []; + var faultItem = JsonSerializer.SerializeToElement(new + { + role = "user", + content = new[] + { + new { type = "input_text", text = faultPrompt } + } + }, ProxyUtils.JsonSerializerOptions); + items.Add(faultItem); + responsesRequest.Input = items; + Logger.LogDebug("Added fault prompt as new item to Responses API input"); + } + catch (JsonException) + { + // If we can't parse as array, append to input as string + responsesRequest.Input = inputElement.GetRawText() + "\n\n" + faultPrompt; + Logger.LogDebug("Modified Responses API input with fault prompt (fallback)"); + } } - Logger.LogDebug("Request is not an OpenAI request."); - return false; + Logger.LogRequest($"Simulating fault {faultName}", MessageType.Chaos, new(e.Session)); + e.Session.SetRequestBodyString(JsonSerializer.Serialize(responsesRequest, ProxyUtils.JsonSerializerOptions)); } - catch (JsonException ex) + else { - Logger.LogDebug(ex, "Failed to deserialize OpenAI request."); - return false; + Logger.LogDebug("Unknown OpenAI request type. Passing request as-is."); } + + await Task.CompletedTask; + + Logger.LogTrace("Left {Name}", nameof(BeforeRequestAsync)); } private (string? Name, string? Prompt) GetFault() diff --git a/DevProxy.Plugins/Behavior/LanguageModelRateLimitingPlugin.cs b/DevProxy.Plugins/Behavior/LanguageModelRateLimitingPlugin.cs index 5dcc2121..df5fd709 100644 --- a/DevProxy.Plugins/Behavior/LanguageModelRateLimitingPlugin.cs +++ b/DevProxy.Plugins/Behavior/LanguageModelRateLimitingPlugin.cs @@ -99,7 +99,7 @@ public override Task BeforeRequestAsync(ProxyRequestArgs e, CancellationToken ca return Task.CompletedTask; } - if (!TryGetOpenAIRequest(request.BodyString, out var openAiRequest)) + if (!OpenAIRequest.TryGetOpenAIRequest(request.BodyString, Logger, out var openAiRequest)) { Logger.LogRequest("Skipping non-OpenAI request", MessageType.Skipped, new(e.Session)); return Task.CompletedTask; @@ -224,7 +224,7 @@ public override Task BeforeResponseAsync(ProxyResponseArgs e, CancellationToken return Task.CompletedTask; } - if (!TryGetOpenAIRequest(request.BodyString, out var openAiRequest)) + if (!OpenAIRequest.TryGetOpenAIRequest(request.BodyString, Logger, out var openAiRequest)) { Logger.LogDebug("Skipping non-OpenAI request"); return Task.CompletedTask; @@ -239,7 +239,18 @@ public override Task BeforeResponseAsync(ProxyResponseArgs e, CancellationToken { try { - var openAiResponse = JsonSerializer.Deserialize(responseBody, ProxyUtils.JsonSerializerOptions); + // Try to parse as Responses API first, then fall back to standard response + OpenAIResponse? openAiResponse = null; + + if (openAiRequest is OpenAIResponsesRequest) + { + openAiResponse = JsonSerializer.Deserialize(responseBody, ProxyUtils.JsonSerializerOptions); + } + else + { + openAiResponse = JsonSerializer.Deserialize(responseBody, ProxyUtils.JsonSerializerOptions); + } + if (openAiResponse?.Usage != null) { var promptTokens = (int)openAiResponse.Usage.PromptTokens; @@ -271,45 +282,6 @@ public override Task BeforeResponseAsync(ProxyResponseArgs e, CancellationToken return Task.CompletedTask; } - private bool TryGetOpenAIRequest(string content, out OpenAIRequest? request) - { - request = null; - - if (string.IsNullOrEmpty(content)) - { - return false; - } - - try - { - Logger.LogDebug("Checking if the request is an OpenAI request..."); - - var rawRequest = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); - - if (rawRequest.TryGetProperty("prompt", out _)) - { - Logger.LogDebug("Request is a completion request"); - request = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); - return true; - } - - if (rawRequest.TryGetProperty("messages", out _)) - { - Logger.LogDebug("Request is a chat completion request"); - request = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); - return true; - } - - Logger.LogDebug("Request is not an OpenAI request."); - return false; - } - catch (JsonException ex) - { - Logger.LogDebug(ex, "Failed to deserialize OpenAI request."); - return false; - } - } - private ThrottlingInfo ShouldThrottle(Request request, string throttlingKey) { var throttleKeyForRequest = BuildThrottleKey(request); diff --git a/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs b/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs index 126db908..37394485 100644 --- a/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs +++ b/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs @@ -350,6 +350,9 @@ private void AddResponseTypeSpecificTags(Activity activity, OpenAIRequest openAi { switch (openAiRequest) { + case OpenAIResponsesRequest: + AddResponsesApiResponseTags(activity, openAiRequest, responseBody); + break; case OpenAIChatCompletionRequest: AddChatCompletionResponseTags(activity, openAiRequest, responseBody); break; @@ -541,6 +544,9 @@ private void AddRequestTypeSpecificTags(Activity activity, OpenAIRequest openAiR { switch (openAiRequest) { + case OpenAIResponsesRequest responsesRequest: + AddResponsesApiRequestTags(activity, responsesRequest); + break; case OpenAIChatCompletionRequest chatRequest: AddChatCompletionRequestTags(activity, chatRequest); break; @@ -914,6 +920,7 @@ private static string GetOperationName(OpenAIRequest request) return request switch { + OpenAIResponsesRequest => "responses", OpenAIChatCompletionRequest => "chat.completions", OpenAICompletionRequest => "completions", OpenAIEmbeddingRequest => "embeddings", @@ -925,6 +932,73 @@ private static string GetOperationName(OpenAIRequest request) }; } + private void AddResponsesApiRequestTags(Activity activity, OpenAIResponsesRequest responsesRequest) + { + Logger.LogTrace("AddResponsesApiRequestTags() called"); + + // OpenLIT + _ = activity.SetTag(SemanticConvention.GEN_AI_OPERATION, "responses") + // OpenTelemetry + .SetTag(SemanticConvention.GEN_AI_OPERATION_NAME, "responses"); + + if (Configuration.IncludePrompt && responsesRequest.Input != null) + { + var inputString = responsesRequest.Input is string str ? str : JsonSerializer.Serialize(responsesRequest.Input, ProxyUtils.JsonSerializerOptions); + _ = activity.SetTag(SemanticConvention.GEN_AI_CONTENT_PROMPT, inputString); + } + + if (responsesRequest.Instructions != null) + { + _ = activity.SetTag("ai.request.instructions", responsesRequest.Instructions); + } + + if (responsesRequest.Modalities != null) + { + _ = activity.SetTag("ai.request.modalities", string.Join(",", responsesRequest.Modalities)); + } + + Logger.LogTrace("AddResponsesApiRequestTags() finished"); + } + + private void AddResponsesApiResponseTags(Activity activity, OpenAIRequest openAIRequest, string responseBody) + { + Logger.LogTrace("AddResponsesApiResponseTags() called"); + + var responsesResponse = JsonSerializer.Deserialize(responseBody, ProxyUtils.JsonSerializerOptions); + if (responsesResponse is null) + { + return; + } + + RecordUsageMetrics(activity, openAIRequest, responsesResponse); + + _ = activity.SetTag(SemanticConvention.GEN_AI_RESPONSE_ID, responsesResponse.Id); + + if (responsesResponse.Status != null) + { + _ = activity.SetTag("ai.response.status", responsesResponse.Status); + } + + // Extract completion text from output items + if (Configuration.IncludeCompletion && responsesResponse.Output != null) + { + var textContent = responsesResponse.Output + .Where(item => item.Type == "message" && item.Content != null) + .SelectMany(item => item.Content!) + .Where(c => c.Type == "output_text") + .Select(c => c.Text) + .Where(t => !string.IsNullOrEmpty(t)) + .LastOrDefault(); + + if (!string.IsNullOrEmpty(textContent)) + { + _ = activity.SetTag(SemanticConvention.GEN_AI_CONTENT_COMPLETION, textContent); + } + } + + Logger.LogTrace("AddResponsesApiResponseTags() finished"); + } + public void Dispose() { _loader?.Dispose(); diff --git a/DevProxy.Plugins/Mocking/OpenAIMockResponsePlugin.cs b/DevProxy.Plugins/Mocking/OpenAIMockResponsePlugin.cs index 464c5ac9..326bab1f 100644 --- a/DevProxy.Plugins/Mocking/OpenAIMockResponsePlugin.cs +++ b/DevProxy.Plugins/Mocking/OpenAIMockResponsePlugin.cs @@ -58,7 +58,7 @@ public override async Task BeforeRequestAsync(ProxyRequestArgs e, CancellationTo return; } - if (!TryGetOpenAIRequest(request.BodyString, out var openAiRequest)) + if (!OpenAIRequest.TryGetOpenAIRequest(request.BodyString, Logger, out var openAiRequest)) { Logger.LogRequest("Skipping non-OpenAI request", MessageType.Skipped, new(e.Session)); return; @@ -95,6 +95,26 @@ public override async Task BeforeRequestAsync(ProxyRequestArgs e, CancellationTo var openAiResponse = lmResponse.ConvertToOpenAIResponse(); SendMockResponse(openAiResponse, lmResponse.RequestUrl ?? string.Empty, e); } + else if (openAiRequest is OpenAIResponsesRequest responsesRequest) + { + // Convert Responses API request to Chat Completion format for local LLM + var messages = ConvertResponsesInputToMessages(responsesRequest); + + if ((await languageModelClient + .GenerateChatCompletionAsync(messages, null, cancellationToken)) is not ILanguageModelCompletionResponse lmResponse) + { + return; + } + if (lmResponse.ErrorMessage is not null) + { + Logger.LogError("Error from local language model: {Error}", lmResponse.ErrorMessage); + return; + } + + // Convert Chat Completion response to Responses API format + var responsesResponse = ConvertToResponsesApiResponse(lmResponse, responsesRequest.Model); + SendMockResponse(responsesResponse, lmResponse.RequestUrl ?? string.Empty, e); + } else { Logger.LogError("Unknown OpenAI request type."); @@ -102,44 +122,127 @@ public override async Task BeforeRequestAsync(ProxyRequestArgs e, CancellationTo Logger.LogTrace("Left {Name}", nameof(BeforeRequestAsync)); } - - private bool TryGetOpenAIRequest(string content, out OpenAIRequest? request) + + private static List ConvertResponsesInputToMessages(OpenAIResponsesRequest responsesRequest) { - request = null; + var messages = new List(); - if (string.IsNullOrEmpty(content)) + // Add instructions as system message if present + if (!string.IsNullOrEmpty(responsesRequest.Instructions)) { - return false; + messages.Add(new OpenAIChatCompletionMessage + { + Role = "system", + Content = responsesRequest.Instructions + }); } - try + if (responsesRequest.Input is string inputString) { - Logger.LogDebug("Checking if the request is an OpenAI request..."); - - var rawRequest = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); - - if (rawRequest.TryGetProperty("prompt", out _)) + // Simple string input + messages.Add(new OpenAIChatCompletionMessage + { + Role = "user", + Content = inputString + }); + } + else if (responsesRequest.Input is JsonElement inputElement) + { + // Try to parse as array of items + try { - Logger.LogDebug("Request is a completion request"); - request = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); - return true; + var items = JsonSerializer.Deserialize>(inputElement.GetRawText(), ProxyUtils.JsonSerializerOptions); + if (items != null) + { + foreach (var item in items) + { + if (item.TryGetProperty("role", out var roleElement) && + item.TryGetProperty("content", out var contentElement)) + { + var role = roleElement.GetString() ?? "user"; + var content = ExtractTextFromContent(contentElement); + + messages.Add(new OpenAIChatCompletionMessage + { + Role = role, + Content = content + }); + } + } + } } - - if (rawRequest.TryGetProperty("messages", out _)) + catch (JsonException) { - Logger.LogDebug("Request is a chat completion request"); - request = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); - return true; + // Fallback: treat as simple text + messages.Add(new OpenAIChatCompletionMessage + { + Role = "user", + Content = inputElement.GetRawText() + }); } + } + + return messages; + } - Logger.LogDebug("Request is not an OpenAI request."); - return false; + private static string ExtractTextFromContent(JsonElement contentElement) + { + if (contentElement.ValueKind == JsonValueKind.String) + { + return contentElement.GetString() ?? string.Empty; + } + + if (contentElement.ValueKind == JsonValueKind.Array) + { + var texts = new List(); + foreach (var item in contentElement.EnumerateArray()) + { + if (item.TryGetProperty("text", out var textElement)) + { + texts.Add(textElement.GetString() ?? string.Empty); + } + } + return string.Join("\n", texts); } - catch (JsonException ex) + + return string.Empty; + } + + private static OpenAIResponsesResponse ConvertToResponsesApiResponse(ILanguageModelCompletionResponse lmResponse, string model) + { + var chatResponse = (OpenAIChatCompletionResponse)lmResponse.ConvertToOpenAIResponse(); + + var outputItems = new List(); + + if (chatResponse.Choices != null && chatResponse.Choices.Any()) { - Logger.LogDebug(ex, "Failed to deserialize OpenAI request."); - return false; + var choice = chatResponse.Choices.First(); + outputItems.Add(new OpenAIResponsesOutputItem + { + Type = "message", + Role = "assistant", + Content = new[] + { + new OpenAIResponsesContentPart + { + Type = "output_text", + Text = choice.Message.Content + } + } + }); } + + return new OpenAIResponsesResponse + { + Id = chatResponse.Id, + Object = "response", + Created = chatResponse.Created, + CreatedAt = chatResponse.Created, + Model = model, + Status = "completed", + Output = outputItems, + Usage = chatResponse.Usage + }; } private void SendMockResponse(OpenAIResponse response, string localLmUrl, ProxyRequestArgs e) where TResponse : OpenAIResponse