From a4a1c670e03638a108e783bad237463f357302e9 Mon Sep 17 00:00:00 2001 From: waldekmastykarz Date: Sun, 12 Oct 2025 15:50:38 +0200 Subject: [PATCH] Adds the HarGeneratorPlugin. Closes #1412 --- .../Generation/HarGeneratorPlugin.cs | 162 ++++++++++++++++++ .../Generation/OpenApiSpecGeneratorPlugin.cs | 8 +- .../Generation/TypeSpecGeneratorPlugin.cs | 14 +- DevProxy.Plugins/Models/Har.cs | 117 +++++++++++++ DevProxy.Plugins/Models/Http.cs | 20 +++ schemas/v1.3.0/hargeneratorplugin.schema.json | 20 +++ 6 files changed, 330 insertions(+), 11 deletions(-) create mode 100644 DevProxy.Plugins/Generation/HarGeneratorPlugin.cs create mode 100644 DevProxy.Plugins/Models/Har.cs create mode 100644 schemas/v1.3.0/hargeneratorplugin.schema.json diff --git a/DevProxy.Plugins/Generation/HarGeneratorPlugin.cs b/DevProxy.Plugins/Generation/HarGeneratorPlugin.cs new file mode 100644 index 00000000..766ef525 --- /dev/null +++ b/DevProxy.Plugins/Generation/HarGeneratorPlugin.cs @@ -0,0 +1,162 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using DevProxy.Abstractions.Plugins; +using DevProxy.Abstractions.Proxy; +using DevProxy.Abstractions.Utils; +using DevProxy.Plugins.Models; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using System.Diagnostics; +using System.Text.Json; +using System.Web; + +namespace DevProxy.Plugins.Generation; + +public sealed class HarGeneratorPluginConfiguration +{ + public bool IncludeSensitiveInformation { get; set; } + public bool IncludeResponse { get; set; } +} + +public sealed class HarGeneratorPlugin( + HttpClient httpClient, + ILogger logger, + ISet urlsToWatch, + IProxyConfiguration proxyConfiguration, + IConfigurationSection pluginConfigurationSection) : + BaseReportingPlugin( + httpClient, + logger, + urlsToWatch, + proxyConfiguration, + pluginConfigurationSection) +{ + public override string Name => nameof(HarGeneratorPlugin); + + public override async Task AfterRecordingStopAsync(RecordingArgs e, CancellationToken cancellationToken) + { + Logger.LogTrace("{Method} called", nameof(AfterRecordingStopAsync)); + + ArgumentNullException.ThrowIfNull(e); + + if (!e.RequestLogs.Any()) + { + Logger.LogDebug("No requests to process"); + return; + } + + Logger.LogInformation("Creating HAR file from recorded requests..."); + + var harFile = new HarFile + { + Log = new HarLog + { + Creator = new HarCreator + { + Name = "DevProxy", + Version = ProxyUtils.ProductVersion + }, + Entries = [.. e.RequestLogs.Where(r => + r.MessageType == MessageType.InterceptedResponse && + r is not null && + r.Context is not null && + r.Context.Session is not null && + ProxyUtils.MatchesUrlToWatch(UrlsToWatch, r.Context.Session.HttpClient.Request.RequestUri.AbsoluteUri)).Select(CreateHarEntry)] + } + }; + + Logger.LogDebug("Serializing HAR file..."); + var harFileJson = JsonSerializer.Serialize(harFile, ProxyUtils.JsonSerializerOptions); + var fileName = $"devproxy-{DateTime.Now:yyyyMMddHHmmss}.har"; + + Logger.LogDebug("Writing HAR file to {FileName}...", fileName); + await File.WriteAllTextAsync(fileName, harFileJson, cancellationToken); + + Logger.LogInformation("Created HAR file {FileName}", fileName); + + StoreReport(fileName, e); + + Logger.LogTrace("Left {Name}", nameof(AfterRecordingStopAsync)); + } + + private string GetHeaderValue(string headerName, string originalValue) + { + if (!Configuration.IncludeSensitiveInformation && + Http.SensitiveHeaders.Contains(headerName, StringComparer.OrdinalIgnoreCase)) + { + return "REDACTED"; + } + return originalValue; + } + + private HarEntry CreateHarEntry(RequestLog log) + { + Debug.Assert(log is not null); + Debug.Assert(log.Context is not null); + + var request = log.Context.Session.HttpClient.Request; + var response = log.Context.Session.HttpClient.Response; + var currentTime = DateTime.UtcNow; + + var entry = new HarEntry + { + StartedDateTime = currentTime.ToString("o"), + Time = 0, // We don't have actual timing data in RequestLog + Request = new HarRequest + { + Method = request.Method, + Url = request.RequestUri?.ToString(), + HttpVersion = $"HTTP/{request.HttpVersion}", + Headers = [.. request.Headers.Select(h => new HarHeader { Name = h.Name, Value = GetHeaderValue(h.Name, string.Join(", ", h.Value)) })], + QueryString = [.. HttpUtility.ParseQueryString(request.RequestUri?.Query ?? "") + .AllKeys + .Where(key => key is not null) + .Select(key => new HarQueryParam { Name = key!, Value = HttpUtility.ParseQueryString(request.RequestUri?.Query ?? "")[key] ?? "" })], + Cookies = [.. request.Headers + .Where(h => string.Equals(h.Name, "Cookie", StringComparison.OrdinalIgnoreCase)) + .Select(h => h.Value) + .SelectMany(v => v.Split(';')) + .Select(c => + { + var parts = c.Split('=', 2); + return new HarCookie { Name = parts[0].Trim(), Value = parts.Length > 1 ? parts[1].Trim() : "" }; + })], + HeadersSize = request.Headers?.ToString()?.Length ?? 0, + BodySize = request.HasBody ? (request.BodyString?.Length ?? 0) : 0, + PostData = request.HasBody ? new HarPostData + { + MimeType = request.ContentType, + Text = request.BodyString ?? "" + } + : null + }, + Response = response is not null ? new HarResponse + { + Status = response.StatusCode, + StatusText = response.StatusDescription, + HttpVersion = $"HTTP/{response.HttpVersion}", + Headers = [.. response.Headers.Select(h => new HarHeader { Name = h.Name, Value = GetHeaderValue(h.Name, string.Join(", ", h.Value)) })], + Cookies = [.. response.Headers + .Where(h => string.Equals(h.Name, "Set-Cookie", StringComparison.OrdinalIgnoreCase)) + .Select(h => h.Value) + .Select(sc => + { + var parts = sc.Split(';')[0].Split('=', 2); + return new HarCookie { Name = parts[0].Trim(), Value = parts.Length > 1 ? parts[1].Trim() : "" }; + })], + Content = new HarContent + { + Size = response.HasBody ? (response.BodyString?.Length ?? 0) : 0, + MimeType = response.ContentType ?? "", + Text = Configuration.IncludeResponse && response.HasBody ? response.BodyString : null + }, + HeadersSize = response.Headers?.ToString()?.Length ?? 0, + BodySize = response.HasBody ? (response.BodyString?.Length ?? 0) : 0 + } : null + }; + + return entry; + } +} \ No newline at end of file diff --git a/DevProxy.Plugins/Generation/OpenApiSpecGeneratorPlugin.cs b/DevProxy.Plugins/Generation/OpenApiSpecGeneratorPlugin.cs index af807c28..f43d6429 100644 --- a/DevProxy.Plugins/Generation/OpenApiSpecGeneratorPlugin.cs +++ b/DevProxy.Plugins/Generation/OpenApiSpecGeneratorPlugin.cs @@ -288,13 +288,13 @@ private void SetParametersFromRequestHeaders(OpenApiOperation operation, HeaderC foreach (var header in headers) { var lowerCaseHeaderName = header.Name.ToLowerInvariant(); - if (Http.StandardHeaders.Contains(lowerCaseHeaderName)) + if (Models.Http.StandardHeaders.Contains(lowerCaseHeaderName)) { Logger.LogDebug(" Skipping standard header {HeaderName}", header.Name); continue; } - if (Http.AuthHeaders.Contains(lowerCaseHeaderName)) + if (Models.Http.AuthHeaders.Contains(lowerCaseHeaderName)) { Logger.LogDebug(" Skipping auth header {HeaderName}", header.Name); continue; @@ -388,13 +388,13 @@ private void SetResponseFromSession(OpenApiOperation operation, Response respons foreach (var header in response.Headers) { var lowerCaseHeaderName = header.Name.ToLowerInvariant(); - if (Http.StandardHeaders.Contains(lowerCaseHeaderName)) + if (Models.Http.StandardHeaders.Contains(lowerCaseHeaderName)) { Logger.LogDebug(" Skipping standard header {HeaderName}", header.Name); continue; } - if (Http.AuthHeaders.Contains(lowerCaseHeaderName)) + if (Models.Http.AuthHeaders.Contains(lowerCaseHeaderName)) { Logger.LogDebug(" Skipping auth header {HeaderName}", header.Name); continue; diff --git a/DevProxy.Plugins/Generation/TypeSpecGeneratorPlugin.cs b/DevProxy.Plugins/Generation/TypeSpecGeneratorPlugin.cs index b95cee82..eb35f989 100644 --- a/DevProxy.Plugins/Generation/TypeSpecGeneratorPlugin.cs +++ b/DevProxy.Plugins/Generation/TypeSpecGeneratorPlugin.cs @@ -175,7 +175,7 @@ private void ProcessAuth(Request httpRequest, TypeSpecFile doc, Operation op) Logger.LogTrace("Entered {Name}", nameof(ProcessAuth)); var authHeaders = httpRequest.Headers - .Where(h => Http.AuthHeaders.Contains(h.Name.ToLowerInvariant())) + .Where(h => Models.Http.AuthHeaders.Contains(h.Name.ToLowerInvariant())) .Select(h => (h.Name, h.Value)); foreach (var (name, value) in authHeaders) @@ -199,7 +199,7 @@ private void ProcessAuth(Request httpRequest, TypeSpecFile doc, Operation op) var query = HttpUtility.ParseQueryString(httpRequest.RequestUri.Query); var authQueryParam = query.AllKeys - .FirstOrDefault(k => k is not null && Http.AuthHeaders.Contains(k.ToLowerInvariant())); + .FirstOrDefault(k => k is not null && Models.Http.AuthHeaders.Contains(k.ToLowerInvariant())); if (authQueryParam is not null) { Logger.LogDebug("Found auth query parameter: {AuthQueryParam}", authQueryParam); @@ -357,8 +357,8 @@ private void ProcessRequestHeaders(Request httpRequest, Operation op) foreach (var header in httpRequest.Headers) { - if (Http.StandardHeaders.Contains(header.Name.ToLowerInvariant()) || - Http.AuthHeaders.Contains(header.Name.ToLowerInvariant())) + if (Models.Http.StandardHeaders.Contains(header.Name.ToLowerInvariant()) || + Models.Http.AuthHeaders.Contains(header.Name.ToLowerInvariant())) { continue; } @@ -400,8 +400,8 @@ private async Task ProcessResponseAsync(Response? httpResponse, TypeSpecFile doc { StatusCode = httpResponse.StatusCode, Headers = httpResponse.Headers - .Where(h => !Http.StandardHeaders.Contains(h.Name.ToLowerInvariant()) && - !Http.AuthHeaders.Contains(h.Name.ToLowerInvariant())) + .Where(h => !Models.Http.StandardHeaders.Contains(h.Name.ToLowerInvariant()) && + !Models.Http.AuthHeaders.Contains(h.Name.ToLowerInvariant())) .ToDictionary(h => h.Name.ToCamelCase(), h => h.Value.GetType().Name) }; @@ -688,7 +688,7 @@ private bool IsParametrizable(string segment) var query = HttpUtility.ParseQueryString(url.Query); foreach (string key in query.Keys) { - if (Http.AuthHeaders.Contains(key.ToLowerInvariant())) + if (Models.Http.AuthHeaders.Contains(key.ToLowerInvariant())) { Logger.LogDebug("Skipping auth header: {Key}", key); continue; diff --git a/DevProxy.Plugins/Models/Har.cs b/DevProxy.Plugins/Models/Har.cs new file mode 100644 index 00000000..e0fb7635 --- /dev/null +++ b/DevProxy.Plugins/Models/Har.cs @@ -0,0 +1,117 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace DevProxy.Plugins.Models; + +internal sealed class HarFile +{ + public HarLog? Log { get; set; } +} + +internal sealed class HarLog +{ + public string Version { get; set; } = "1.2"; + public HarCreator Creator { get; set; } = new(); + public List Entries { get; set; } = []; +} + +internal sealed class HarCreator +{ + public string? Name { get; set; } + public string? Version { get; set; } +} + +internal sealed class HarEntry +{ + public string? StartedDateTime { get; set; } + public double Time { get; set; } + public HarRequest? Request { get; set; } + public HarResponse? Response { get; set; } + public HarCache Cache { get; set; } = new(); + public HarTimings Timings { get; set; } = new(); +} + +internal sealed class HarRequest +{ + public string? Method { get; set; } + public string? Url { get; set; } + public string? HttpVersion { get; set; } + public List Headers { get; set; } = []; + public List QueryString { get; set; } = []; + public List Cookies { get; set; } = []; + public long HeadersSize { get; set; } + public long BodySize { get; set; } + public HarPostData? PostData { get; set; } +} + +internal sealed class HarResponse +{ + public int Status { get; set; } + public string? StatusText { get; set; } + public string? HttpVersion { get; set; } + public List Headers { get; set; } = []; + public List Cookies { get; set; } = []; + public HarContent Content { get; set; } = new(); + public string RedirectURL { get; set; } = ""; + public long HeadersSize { get; set; } + public long BodySize { get; set; } +} + +internal sealed class HarHeader +{ + public string? Name { get; set; } + public string? Value { get; set; } +} + +internal sealed class HarQueryParam +{ + public string? Name { get; set; } + public string? Value { get; set; } +} + +internal sealed class HarCookie +{ + public string? Name { get; set; } + public string? Value { get; set; } + public string? Path { get; set; } + public string? Domain { get; set; } + public string? Expires { get; set; } + public bool? HttpOnly { get; set; } + public bool? Secure { get; set; } +} + +internal sealed class HarPostData +{ + public string? MimeType { get; set; } + public string? Text { get; set; } + public List? Params { get; set; } +} + +internal sealed class HarParam +{ + public string? Name { get; set; } + public string? Value { get; set; } + public string? FileName { get; set; } + public string? ContentType { get; set; } +} + +internal sealed class HarContent +{ + public long Size { get; set; } + public string MimeType { get; set; } = ""; + public string? Text { get; set; } + public string? Encoding { get; set; } +} + +internal sealed class HarCache +{ + // Minimal - can be expanded if needed +} + +internal sealed class HarTimings +{ + public double Send { get; set; } + public double Wait { get; set; } + public double Receive { get; set; } +} \ No newline at end of file diff --git a/DevProxy.Plugins/Models/Http.cs b/DevProxy.Plugins/Models/Http.cs index 610c48bd..a7cf0281 100644 --- a/DevProxy.Plugins/Models/Http.cs +++ b/DevProxy.Plugins/Models/Http.cs @@ -2,6 +2,8 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +namespace DevProxy.Plugins.Models; + internal static class Http { // from: https://github.com/jonluca/har-to-openapi/blob/0d44409162c0a127cdaccd60b0a270ecd361b829/src/utils/headers.ts @@ -237,4 +239,22 @@ internal static class Http "apikey", "code" ]; + + internal static readonly string[] SensitiveHeaders = + [ + "authorization", + "cookie", + "from", + "proxy-authenticate", + "proxy-authorization", + "set-cookie", + "www-authenticate", + "x-api-key", + "x-auth-token", + "x-csrf-token", + "x-forwarded-for", + "x-real-ip", + "x-session-token", + "x-xsrf-token" + ]; } \ No newline at end of file diff --git a/schemas/v1.3.0/hargeneratorplugin.schema.json b/schemas/v1.3.0/hargeneratorplugin.schema.json new file mode 100644 index 00000000..a7aa5b5e --- /dev/null +++ b/schemas/v1.3.0/hargeneratorplugin.schema.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Dev Proxy HarGeneratorPlugin config schema", + "type": "object", + "properties": { + "$schema": { + "type": "string", + "description": "Reference to the JSON schema definition." + }, + "includeSensitiveInformation": { + "type": "boolean", + "description": "Determines whether to include sensitive information (such as authentication headers, and cookies) in the generated HAR file. When set to false, sensitive information will be redacted. Default: false." + }, + "includeResponse": { + "type": "boolean", + "description": "Determines whether to include HTTP response body in the generated HAR file. When set to false, only request information will be included. Default: false." + } + }, + "additionalProperties": false +}