From 2017747ab7f862183e5b645bc520a5ef2be63544 Mon Sep 17 00:00:00 2001 From: Artem Azaraev Date: Mon, 17 Nov 2025 14:04:32 +0200 Subject: [PATCH 01/10] Use the empty event args instance --- DevProxy/Proxy/ProxyStateController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DevProxy/Proxy/ProxyStateController.cs b/DevProxy/Proxy/ProxyStateController.cs index 1f418abe..93e6d6fb 100644 --- a/DevProxy/Proxy/ProxyStateController.cs +++ b/DevProxy/Proxy/ProxyStateController.cs @@ -69,7 +69,7 @@ public async Task StopRecordingAsync(CancellationToken cancellationToken) public async Task MockRequestAsync(CancellationToken cancellationToken) { - var eventArgs = new EventArgs(); + var eventArgs = EventArgs.Empty; foreach (var plugin in _plugins.Where(p => p.Enabled)) { From 9f06efeef93ec9ad89309b98a6ad467c3596d4e5 Mon Sep 17 00:00:00 2001 From: Artem Azaraev Date: Mon, 17 Nov 2025 14:07:28 +0200 Subject: [PATCH 02/10] Add AfterRecordingStartAsync handler/event --- DevProxy.Abstractions/Plugins/BasePlugin.cs | 5 +++++ DevProxy.Abstractions/Plugins/IPlugin.cs | 1 + DevProxy/ApiControllers/ProxyController.cs | 2 +- DevProxy/Proxy/IProxyStateController.cs | 2 +- DevProxy/Proxy/ProxyEngine.cs | 8 ++++---- DevProxy/Proxy/ProxyStateController.cs | 15 ++++++++++++++- 6 files changed, 26 insertions(+), 7 deletions(-) diff --git a/DevProxy.Abstractions/Plugins/BasePlugin.cs b/DevProxy.Abstractions/Plugins/BasePlugin.cs index d093f9da..b226b394 100644 --- a/DevProxy.Abstractions/Plugins/BasePlugin.cs +++ b/DevProxy.Abstractions/Plugins/BasePlugin.cs @@ -55,6 +55,11 @@ public virtual Task AfterRequestLogAsync(RequestLogArgs e, CancellationToken can return Task.CompletedTask; } + public virtual Task AfterRecordingStartAsync(EventArgs e, CancellationToken cancellationToken) + { + return Task.CompletedTask; + } + public virtual Task AfterRecordingStopAsync(RecordingArgs e, CancellationToken cancellationToken) { return Task.CompletedTask; diff --git a/DevProxy.Abstractions/Plugins/IPlugin.cs b/DevProxy.Abstractions/Plugins/IPlugin.cs index a86d6d71..3a02565b 100644 --- a/DevProxy.Abstractions/Plugins/IPlugin.cs +++ b/DevProxy.Abstractions/Plugins/IPlugin.cs @@ -22,6 +22,7 @@ public interface IPlugin Task BeforeResponseAsync(ProxyResponseArgs e, CancellationToken cancellationToken); Task AfterResponseAsync(ProxyResponseArgs e, CancellationToken cancellationToken); Task AfterRequestLogAsync(RequestLogArgs e, CancellationToken cancellationToken); + Task AfterRecordingStartAsync(EventArgs e, CancellationToken cancellationToken); Task AfterRecordingStopAsync(RecordingArgs e, CancellationToken cancellationToken); Task MockRequestAsync(EventArgs e, CancellationToken cancellationToken); } diff --git a/DevProxy/ApiControllers/ProxyController.cs b/DevProxy/ApiControllers/ProxyController.cs index cec82532..71ad10b0 100644 --- a/DevProxy/ApiControllers/ProxyController.cs +++ b/DevProxy/ApiControllers/ProxyController.cs @@ -42,7 +42,7 @@ public async Task SetAsync([FromBody] ProxyInfo proxyInfo, Cancel { if (proxyInfo.Recording.Value) { - _proxyStateController.StartRecording(); + await _proxyStateController.StartRecordingAsync(cancellationToken); } else { diff --git a/DevProxy/Proxy/IProxyStateController.cs b/DevProxy/Proxy/IProxyStateController.cs index 87a98103..b05c2223 100644 --- a/DevProxy/Proxy/IProxyStateController.cs +++ b/DevProxy/Proxy/IProxyStateController.cs @@ -9,7 +9,7 @@ public interface IProxyStateController #pragma warning restore CA1515 { IProxyState ProxyState { get; } - void StartRecording(); + Task StartRecordingAsync(CancellationToken cancellationToken); Task StopRecordingAsync(CancellationToken cancellationToken); Task MockRequestAsync(CancellationToken cancellationToken); void StopProxy(); diff --git a/DevProxy/Proxy/ProxyEngine.cs b/DevProxy/Proxy/ProxyEngine.cs index 04c1498c..9f0a50cb 100755 --- a/DevProxy/Proxy/ProxyEngine.cs +++ b/DevProxy/Proxy/ProxyEngine.cs @@ -147,7 +147,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) if (_config.Record) { - StartRecording(); + await StartRecordingAsync(stoppingToken); } if (_config.TimeoutSeconds.HasValue) @@ -210,7 +210,7 @@ private async Task ReadKeysAsync(CancellationToken cancellationToken) #pragma warning restore IDE0010 { case ConsoleKey.R: - StartRecording(); + await StartRecordingAsync(cancellationToken); break; case ConsoleKey.S: await StopRecordingAsync(cancellationToken); @@ -225,14 +225,14 @@ private async Task ReadKeysAsync(CancellationToken cancellationToken) } } - private void StartRecording() + private async Task StartRecordingAsync(CancellationToken cancellationToken) { if (_proxyController.ProxyState.IsRecording) { return; } - _proxyController.StartRecording(); + await _proxyController.StartRecordingAsync(cancellationToken); } private async Task StopRecordingAsync(CancellationToken cancellationToken) diff --git a/DevProxy/Proxy/ProxyStateController.cs b/DevProxy/Proxy/ProxyStateController.cs index 93e6d6fb..49f6283b 100644 --- a/DevProxy/Proxy/ProxyStateController.cs +++ b/DevProxy/Proxy/ProxyStateController.cs @@ -23,7 +23,7 @@ sealed class ProxyStateController( private readonly ILogger _logger = logger; private ExceptionHandler ExceptionHandler => ex => _logger.LogError(ex, "An error occurred in a plugin"); - public void StartRecording() + public async Task StartRecordingAsync(CancellationToken cancellationToken) { if (ProxyState.IsRecording) { @@ -32,6 +32,19 @@ public void StartRecording() ProxyState.IsRecording = true; PrintRecordingIndicator(ProxyState.IsRecording); + + var eventArgs = EventArgs.Empty; + foreach (var plugin in _plugins.Where(p => p.Enabled)) + { + try + { + await plugin.AfterRecordingStartAsync(eventArgs, cancellationToken); + } + catch (Exception ex) + { + ExceptionHandler(ex); + } + } } public async Task StopRecordingAsync(CancellationToken cancellationToken) From a7c25cc1d367509d0d109aeda5bc371176c29baa Mon Sep 17 00:00:00 2001 From: Artem Azaraev Date: Mon, 17 Nov 2025 14:08:46 +0200 Subject: [PATCH 03/10] Add tracking flag of the recording start --- .../Inspection/OpenAITelemetryPlugin.cs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs b/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs index 126db908..7a22cc01 100644 --- a/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs +++ b/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs @@ -67,7 +67,9 @@ public sealed class OpenAITelemetryPlugin( private LanguageModelPricesLoader? _loader; private MeterProvider? _meterProvider; private TracerProvider? _tracerProvider; + private readonly ConcurrentDictionary> _modelUsage = []; + private bool _isRecording = false; public override string Name => nameof(OpenAITelemetryPlugin); @@ -192,6 +194,16 @@ public override Task AfterResponseAsync(ProxyResponseArgs e, CancellationToken c return Task.CompletedTask; } + public override Task AfterRecordingStartAsync(EventArgs e, CancellationToken cancellationToken) + { + Logger.LogTrace("{Method} called", nameof(AfterRecordingStartAsync)); + + _isRecording = true; + + Logger.LogTrace("Left {Name}", nameof(AfterRecordingStartAsync)); + return Task.CompletedTask; + } + public override Task AfterRecordingStopAsync(RecordingArgs e, CancellationToken cancellationToken) { Logger.LogTrace("{Method} called", nameof(AfterRecordingStopAsync)); @@ -206,6 +218,7 @@ public override Task AfterRecordingStopAsync(RecordingArgs e, CancellationToken }; StoreReport(report, e); + _isRecording = false; _modelUsage.Clear(); Logger.LogTrace("Left {Name}", nameof(AfterRecordingStopAsync)); From 6f86cdb478b68265d9a3a46ea0281fa7a66be0a1 Mon Sep 17 00:00:00 2001 From: Artem Azaraev Date: Mon, 17 Nov 2025 14:09:04 +0200 Subject: [PATCH 04/10] Collect model usage info if the recording is on --- DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs b/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs index 7a22cc01..6ff5ba9e 100644 --- a/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs +++ b/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs @@ -869,8 +869,12 @@ private void RecordUsageMetrics(Activity activity, OpenAIRequest request, OpenAI CompletionTokens = usage.CompletionTokens, CachedTokens = usage.PromptTokensDetails?.CachedTokens ?? 0L }; - var usagePerModel = _modelUsage.GetOrAdd(response.Model, model => []); - usagePerModel.Add(reportModelUsageInformation); + + if (_isRecording) + { + var usagePerModel = _modelUsage.GetOrAdd(response.Model, model => []); + usagePerModel.Add(reportModelUsageInformation); + } if (!Configuration.IncludeCosts || Configuration.Prices is null) { From d06fa7cc05313912738ab093d1acf2676ea8f3f5 Mon Sep 17 00:00:00 2001 From: Artem Azaraev Date: Mon, 17 Nov 2025 15:40:30 +0200 Subject: [PATCH 05/10] Remove redundant initialization --- DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs b/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs index 6ff5ba9e..f9146a90 100644 --- a/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs +++ b/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs @@ -69,7 +69,7 @@ public sealed class OpenAITelemetryPlugin( private TracerProvider? _tracerProvider; private readonly ConcurrentDictionary> _modelUsage = []; - private bool _isRecording = false; + private bool _isRecording; public override string Name => nameof(OpenAITelemetryPlugin); From 0a30e8f03d8e48485084ba940542e85ae09cdd1d Mon Sep 17 00:00:00 2001 From: Artem Azaraev Date: Tue, 25 Nov 2025 07:53:38 +0200 Subject: [PATCH 06/10] Revert all related new AfterRecordingStartAsync handler --- DevProxy.Abstractions/Plugins/BasePlugin.cs | 5 ----- DevProxy.Abstractions/Plugins/IPlugin.cs | 1 - .../Inspection/OpenAITelemetryPlugin.cs | 21 ++----------------- DevProxy/ApiControllers/ProxyController.cs | 2 +- DevProxy/Proxy/IProxyStateController.cs | 2 +- DevProxy/Proxy/ProxyEngine.cs | 8 +++---- DevProxy/Proxy/ProxyStateController.cs | 15 +------------ 7 files changed, 9 insertions(+), 45 deletions(-) diff --git a/DevProxy.Abstractions/Plugins/BasePlugin.cs b/DevProxy.Abstractions/Plugins/BasePlugin.cs index b226b394..d093f9da 100644 --- a/DevProxy.Abstractions/Plugins/BasePlugin.cs +++ b/DevProxy.Abstractions/Plugins/BasePlugin.cs @@ -55,11 +55,6 @@ public virtual Task AfterRequestLogAsync(RequestLogArgs e, CancellationToken can return Task.CompletedTask; } - public virtual Task AfterRecordingStartAsync(EventArgs e, CancellationToken cancellationToken) - { - return Task.CompletedTask; - } - public virtual Task AfterRecordingStopAsync(RecordingArgs e, CancellationToken cancellationToken) { return Task.CompletedTask; diff --git a/DevProxy.Abstractions/Plugins/IPlugin.cs b/DevProxy.Abstractions/Plugins/IPlugin.cs index 3a02565b..a86d6d71 100644 --- a/DevProxy.Abstractions/Plugins/IPlugin.cs +++ b/DevProxy.Abstractions/Plugins/IPlugin.cs @@ -22,7 +22,6 @@ public interface IPlugin Task BeforeResponseAsync(ProxyResponseArgs e, CancellationToken cancellationToken); Task AfterResponseAsync(ProxyResponseArgs e, CancellationToken cancellationToken); Task AfterRequestLogAsync(RequestLogArgs e, CancellationToken cancellationToken); - Task AfterRecordingStartAsync(EventArgs e, CancellationToken cancellationToken); Task AfterRecordingStopAsync(RecordingArgs e, CancellationToken cancellationToken); Task MockRequestAsync(EventArgs e, CancellationToken cancellationToken); } diff --git a/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs b/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs index f9146a90..126db908 100644 --- a/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs +++ b/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs @@ -67,9 +67,7 @@ public sealed class OpenAITelemetryPlugin( private LanguageModelPricesLoader? _loader; private MeterProvider? _meterProvider; private TracerProvider? _tracerProvider; - private readonly ConcurrentDictionary> _modelUsage = []; - private bool _isRecording; public override string Name => nameof(OpenAITelemetryPlugin); @@ -194,16 +192,6 @@ public override Task AfterResponseAsync(ProxyResponseArgs e, CancellationToken c return Task.CompletedTask; } - public override Task AfterRecordingStartAsync(EventArgs e, CancellationToken cancellationToken) - { - Logger.LogTrace("{Method} called", nameof(AfterRecordingStartAsync)); - - _isRecording = true; - - Logger.LogTrace("Left {Name}", nameof(AfterRecordingStartAsync)); - return Task.CompletedTask; - } - public override Task AfterRecordingStopAsync(RecordingArgs e, CancellationToken cancellationToken) { Logger.LogTrace("{Method} called", nameof(AfterRecordingStopAsync)); @@ -218,7 +206,6 @@ public override Task AfterRecordingStopAsync(RecordingArgs e, CancellationToken }; StoreReport(report, e); - _isRecording = false; _modelUsage.Clear(); Logger.LogTrace("Left {Name}", nameof(AfterRecordingStopAsync)); @@ -869,12 +856,8 @@ private void RecordUsageMetrics(Activity activity, OpenAIRequest request, OpenAI CompletionTokens = usage.CompletionTokens, CachedTokens = usage.PromptTokensDetails?.CachedTokens ?? 0L }; - - if (_isRecording) - { - var usagePerModel = _modelUsage.GetOrAdd(response.Model, model => []); - usagePerModel.Add(reportModelUsageInformation); - } + var usagePerModel = _modelUsage.GetOrAdd(response.Model, model => []); + usagePerModel.Add(reportModelUsageInformation); if (!Configuration.IncludeCosts || Configuration.Prices is null) { diff --git a/DevProxy/ApiControllers/ProxyController.cs b/DevProxy/ApiControllers/ProxyController.cs index 71ad10b0..cec82532 100644 --- a/DevProxy/ApiControllers/ProxyController.cs +++ b/DevProxy/ApiControllers/ProxyController.cs @@ -42,7 +42,7 @@ public async Task SetAsync([FromBody] ProxyInfo proxyInfo, Cancel { if (proxyInfo.Recording.Value) { - await _proxyStateController.StartRecordingAsync(cancellationToken); + _proxyStateController.StartRecording(); } else { diff --git a/DevProxy/Proxy/IProxyStateController.cs b/DevProxy/Proxy/IProxyStateController.cs index b05c2223..87a98103 100644 --- a/DevProxy/Proxy/IProxyStateController.cs +++ b/DevProxy/Proxy/IProxyStateController.cs @@ -9,7 +9,7 @@ public interface IProxyStateController #pragma warning restore CA1515 { IProxyState ProxyState { get; } - Task StartRecordingAsync(CancellationToken cancellationToken); + void StartRecording(); Task StopRecordingAsync(CancellationToken cancellationToken); Task MockRequestAsync(CancellationToken cancellationToken); void StopProxy(); diff --git a/DevProxy/Proxy/ProxyEngine.cs b/DevProxy/Proxy/ProxyEngine.cs index 9f0a50cb..04c1498c 100755 --- a/DevProxy/Proxy/ProxyEngine.cs +++ b/DevProxy/Proxy/ProxyEngine.cs @@ -147,7 +147,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) if (_config.Record) { - await StartRecordingAsync(stoppingToken); + StartRecording(); } if (_config.TimeoutSeconds.HasValue) @@ -210,7 +210,7 @@ private async Task ReadKeysAsync(CancellationToken cancellationToken) #pragma warning restore IDE0010 { case ConsoleKey.R: - await StartRecordingAsync(cancellationToken); + StartRecording(); break; case ConsoleKey.S: await StopRecordingAsync(cancellationToken); @@ -225,14 +225,14 @@ private async Task ReadKeysAsync(CancellationToken cancellationToken) } } - private async Task StartRecordingAsync(CancellationToken cancellationToken) + private void StartRecording() { if (_proxyController.ProxyState.IsRecording) { return; } - await _proxyController.StartRecordingAsync(cancellationToken); + _proxyController.StartRecording(); } private async Task StopRecordingAsync(CancellationToken cancellationToken) diff --git a/DevProxy/Proxy/ProxyStateController.cs b/DevProxy/Proxy/ProxyStateController.cs index 49f6283b..93e6d6fb 100644 --- a/DevProxy/Proxy/ProxyStateController.cs +++ b/DevProxy/Proxy/ProxyStateController.cs @@ -23,7 +23,7 @@ sealed class ProxyStateController( private readonly ILogger _logger = logger; private ExceptionHandler ExceptionHandler => ex => _logger.LogError(ex, "An error occurred in a plugin"); - public async Task StartRecordingAsync(CancellationToken cancellationToken) + public void StartRecording() { if (ProxyState.IsRecording) { @@ -32,19 +32,6 @@ public async Task StartRecordingAsync(CancellationToken cancellationToken) ProxyState.IsRecording = true; PrintRecordingIndicator(ProxyState.IsRecording); - - var eventArgs = EventArgs.Empty; - foreach (var plugin in _plugins.Where(p => p.Enabled)) - { - try - { - await plugin.AfterRecordingStartAsync(eventArgs, cancellationToken); - } - catch (Exception ex) - { - ExceptionHandler(ex); - } - } } public async Task StopRecordingAsync(CancellationToken cancellationToken) From 2dd0a2326880cf7563f1e4e9a53bdd8f3ae4205f Mon Sep 17 00:00:00 2001 From: Artem Azaraev Date: Tue, 25 Nov 2025 07:58:19 +0200 Subject: [PATCH 07/10] Add param null check --- DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs b/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs index 126db908..d02bebf3 100644 --- a/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs +++ b/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs @@ -196,6 +196,8 @@ public override Task AfterRecordingStopAsync(RecordingArgs e, CancellationToken { Logger.LogTrace("{Method} called", nameof(AfterRecordingStopAsync)); + ArgumentNullException.ThrowIfNull(e); + var report = new OpenAITelemetryPluginReport { Application = Configuration.Application, From 5b3a74d75a89f229eb21489c845873c6e7c641ba Mon Sep 17 00:00:00 2001 From: Artem Azaraev Date: Tue, 25 Nov 2025 08:01:45 +0200 Subject: [PATCH 08/10] Remove modelUsage storage to collect data afterward --- DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs b/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs index d02bebf3..3f958402 100644 --- a/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs +++ b/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs @@ -16,7 +16,6 @@ using OpenTelemetry.Metrics; using OpenTelemetry.Resources; using OpenTelemetry.Trace; -using System.Collections.Concurrent; using System.Diagnostics; using System.Diagnostics.Metrics; using System.Text.Json; @@ -67,7 +66,6 @@ public sealed class OpenAITelemetryPlugin( private LanguageModelPricesLoader? _loader; private MeterProvider? _meterProvider; private TracerProvider? _tracerProvider; - private readonly ConcurrentDictionary> _modelUsage = []; public override string Name => nameof(OpenAITelemetryPlugin); @@ -204,11 +202,9 @@ public override Task AfterRecordingStopAsync(RecordingArgs e, CancellationToken Environment = Configuration.Environment, Currency = Configuration.Currency, IncludeCosts = Configuration.IncludeCosts, - ModelUsage = _modelUsage.ToDictionary() }; StoreReport(report, e); - _modelUsage.Clear(); Logger.LogTrace("Left {Name}", nameof(AfterRecordingStopAsync)); return Task.CompletedTask; @@ -858,8 +854,6 @@ private void RecordUsageMetrics(Activity activity, OpenAIRequest request, OpenAI CompletionTokens = usage.CompletionTokens, CachedTokens = usage.PromptTokensDetails?.CachedTokens ?? 0L }; - var usagePerModel = _modelUsage.GetOrAdd(response.Model, model => []); - usagePerModel.Add(reportModelUsageInformation); if (!Configuration.IncludeCosts || Configuration.Prices is null) { From 83409c6a8d9f3957c7f0e5ba4a415a2987ccaad1 Mon Sep 17 00:00:00 2001 From: Artem Azaraev Date: Tue, 25 Nov 2025 14:00:43 +0200 Subject: [PATCH 09/10] Add modelUsage collection from RequestLogs after recording is stopped --- .../Inspection/OpenAITelemetryPlugin.cs | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs b/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs index 3f958402..5c66f8a7 100644 --- a/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs +++ b/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs @@ -11,6 +11,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; using OpenTelemetry; using OpenTelemetry.Exporter; using OpenTelemetry.Metrics; @@ -202,6 +203,7 @@ public override Task AfterRecordingStopAsync(RecordingArgs e, CancellationToken Environment = Configuration.Environment, Currency = Configuration.Currency, IncludeCosts = Configuration.IncludeCosts, + ModelUsage = GetOpenAIModelUsage(e.RequestLogs) }; StoreReport(report, e); @@ -901,6 +903,100 @@ private void RecordUsageMetrics(Activity activity, OpenAIRequest request, OpenAI Logger.LogTrace("RecordUsageMetrics() finished"); } + private Dictionary> GetOpenAIModelUsage(IEnumerable requestLogs) + { + var modelUsage = new Dictionary>(); + var openAIRequestLogs = requestLogs.Where(r => + r is not null && + r.Context is not null && + r.Context.Session is not null && + r.MessageType == MessageType.InterceptedResponse && + string.Equals("POST", r.Context.Session.HttpClient.Request.Method, StringComparison.OrdinalIgnoreCase) && + r.Context.Session.HttpClient.Response.StatusCode >= 200 && + r.Context.Session.HttpClient.Response.StatusCode < 300 && + r.Context.Session.HttpClient.Response.HasBody && + !string.IsNullOrEmpty(r.Context.Session.HttpClient.Response.BodyString) && + ProxyUtils.MatchesUrlToWatch(UrlsToWatch, r.Context.Session.HttpClient.Request.RequestUri.AbsoluteUri) && + OpenAIRequest.TryGetOpenAIRequest(r.Context.Session.HttpClient.Request.BodyString, NullLogger.Instance, out var openAiRequest) && + openAiRequest is not null + ); + + foreach (var requestLog in openAIRequestLogs) + { + try + { + var response = JsonSerializer.Deserialize(requestLog.Context!.Session.HttpClient.Response.BodyString, ProxyUtils.JsonSerializerOptions); + if (response is null) + { + continue; + } + + var reportModelUsageInfo = GetReportModelUsageInfo(response); + if (modelUsage.TryGetValue(response.Model, out var usagePerModel)) + { + usagePerModel.AddRange(reportModelUsageInfo); + } + else + { + modelUsage.Add(response.Model, reportModelUsageInfo); + } + } + catch (JsonException ex) + { + Logger.LogError(ex, "Failed to deserialize OpenAI response"); + } + } + + return modelUsage; + } + + private List GetReportModelUsageInfo(OpenAIResponse response) + { + Logger.LogTrace("GetReportModelUsageInfo() called"); + var usagePerModel = new List(); + var usage = response.Usage; + if (usage is null) + { + return usagePerModel; + } + + var reportModelUsageInformation = new OpenAITelemetryPluginReportModelUsageInformation + { + Model = response.Model, + PromptTokens = usage.PromptTokens, + CompletionTokens = usage.CompletionTokens, + CachedTokens = usage.PromptTokensDetails?.CachedTokens ?? 0L + }; + usagePerModel.Add(reportModelUsageInformation); + + if (!Configuration.IncludeCosts || Configuration.Prices is null) + { + Logger.LogDebug("Cost tracking is disabled or prices data is not available"); + return usagePerModel; + } + + if (string.IsNullOrEmpty(response.Model)) + { + Logger.LogDebug("Response model is empty or null"); + return usagePerModel; + } + + var (inputCost, outputCost) = Configuration.Prices.CalculateCost(response.Model, usage.PromptTokens, usage.CompletionTokens); + + if (inputCost > 0) + { + var totalCost = inputCost + outputCost; + reportModelUsageInformation.Cost = totalCost; + } + else + { + Logger.LogDebug("Input cost is zero, skipping cost metrics recording"); + } + + Logger.LogTrace("GetReportModelUsageInfo() finished"); + return usagePerModel; + } + private static string GetOperationName(OpenAIRequest request) { if (request == null) From 3d1418c492f8ffc3271b051915d01c4aaee51970 Mon Sep 17 00:00:00 2001 From: Waldek Mastykarz Date: Fri, 28 Nov 2025 13:47:15 +0100 Subject: [PATCH 10/10] Removes unused object --- DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs b/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs index 5c66f8a7..7ff4a0b1 100644 --- a/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs +++ b/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs @@ -849,14 +849,6 @@ private void RecordUsageMetrics(Activity activity, OpenAIRequest request, OpenAI .SetTag(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS, usage.CompletionTokens) .SetTag(SemanticConvention.GEN_AI_USAGE_TOTAL_TOKENS, usage.TotalTokens); - var reportModelUsageInformation = new OpenAITelemetryPluginReportModelUsageInformation - { - Model = response.Model, - PromptTokens = usage.PromptTokens, - CompletionTokens = usage.CompletionTokens, - CachedTokens = usage.PromptTokensDetails?.CachedTokens ?? 0L - }; - if (!Configuration.IncludeCosts || Configuration.Prices is null) { Logger.LogDebug("Cost tracking is disabled or prices data is not available"); @@ -893,7 +885,6 @@ private void RecordUsageMetrics(Activity activity, OpenAIRequest request, OpenAI new(SemanticConvention.GEN_AI_REQUEST_MODEL, request.Model), new(SemanticConvention.GEN_AI_RESPONSE_MODEL, response.Model) ]); - reportModelUsageInformation.Cost = totalCost; } else {