Adds the HarGeneratorPlugin. Closes #1412

DevProxy.Plugins/Generation/HarGeneratorPlugin.cs

@@ -0,0 +1,162 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using DevProxy.Abstractions.Plugins;
+using DevProxy.Abstractions.Proxy;
+using DevProxy.Abstractions.Utils;
+using DevProxy.Plugins.Models;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
+using System.Diagnostics;
+using System.Text.Json;
+using System.Web;
+
+namespace DevProxy.Plugins.Generation;
+
+public sealed class HarGeneratorPluginConfiguration
+{
+    public bool IncludeSensitiveInformation { get; set; }
+    public bool IncludeResponse { get; set; }
+}
+
+public sealed class HarGeneratorPlugin(
+    HttpClient httpClient,
+    ILogger<HarGeneratorPlugin> logger,
+    ISet<UrlToWatch> urlsToWatch,
+    IProxyConfiguration proxyConfiguration,
+    IConfigurationSection pluginConfigurationSection) :
+    BaseReportingPlugin<HarGeneratorPluginConfiguration>(
+        httpClient,
+        logger,
+        urlsToWatch,
+        proxyConfiguration,
+        pluginConfigurationSection)
+{
+    public override string Name => nameof(HarGeneratorPlugin);
+
+    public override async Task AfterRecordingStopAsync(RecordingArgs e, CancellationToken cancellationToken)
+    {
+        Logger.LogTrace("{Method} called", nameof(AfterRecordingStopAsync));
+
+        ArgumentNullException.ThrowIfNull(e);
+
+        if (!e.RequestLogs.Any())
+        {
+            Logger.LogDebug("No requests to process");
+            return;
+        }
+
+        Logger.LogInformation("Creating HAR file from recorded requests...");
+
+        var harFile = new HarFile
+        {
+            Log = new HarLog
+            {
+                Creator = new HarCreator
+                {
+                    Name = "DevProxy",
+                    Version = ProxyUtils.ProductVersion
+                },
+                Entries = [.. e.RequestLogs.Where(r =>
+                    r.MessageType == MessageType.InterceptedResponse &&
+                    r is not null &&
+                    r.Context is not null &&
+                    r.Context.Session is not null &&
+                    ProxyUtils.MatchesUrlToWatch(UrlsToWatch, r.Context.Session.HttpClient.Request.RequestUri.AbsoluteUri)).Select(CreateHarEntry)]
+            }
+        };
+
+        Logger.LogDebug("Serializing HAR file...");
+        var harFileJson = JsonSerializer.Serialize(harFile, ProxyUtils.JsonSerializerOptions);
+        var fileName = $"devproxy-{DateTime.Now:yyyyMMddHHmmss}.har";
+
+        Logger.LogDebug("Writing HAR file to {FileName}...", fileName);
+        await File.WriteAllTextAsync(fileName, harFileJson, cancellationToken);
+
+        Logger.LogInformation("Created HAR file {FileName}", fileName);
+
+        StoreReport(fileName, e);
+
+        Logger.LogTrace("Left {Name}", nameof(AfterRecordingStopAsync));
+    }
+
+    private string GetHeaderValue(string headerName, string originalValue)
+    {
+        if (!Configuration.IncludeSensitiveInformation &&
+            Http.SensitiveHeaders.Contains(headerName, StringComparer.OrdinalIgnoreCase))
+        {
+            return "REDACTED";
+        }
+        return originalValue;
+    }
+
+    private HarEntry CreateHarEntry(RequestLog log)
+    {
+        Debug.Assert(log is not null);
+        Debug.Assert(log.Context is not null);
+
+        var request = log.Context.Session.HttpClient.Request;
+        var response = log.Context.Session.HttpClient.Response;
+        var currentTime = DateTime.UtcNow;
+
+        var entry = new HarEntry
+        {
+            StartedDateTime = currentTime.ToString("o"),
+            Time = 0, // We don't have actual timing data in RequestLog
+            Request = new HarRequest
+            {
+                Method = request.Method,
+                Url = request.RequestUri?.ToString(),
+                HttpVersion = $"HTTP/{request.HttpVersion}",
+                Headers = [.. request.Headers.Select(h => new HarHeader { Name = h.Name, Value = GetHeaderValue(h.Name, string.Join(", ", h.Value)) })],
+                QueryString = [.. HttpUtility.ParseQueryString(request.RequestUri?.Query ?? "")
+                    .AllKeys
+                    .Where(key => key is not null)
+                    .Select(key => new HarQueryParam { Name = key!, Value = HttpUtility.ParseQueryString(request.RequestUri?.Query ?? "")[key] ?? "" })],
+                Cookies = [.. request.Headers
+                    .Where(h => string.Equals(h.Name, "Cookie", StringComparison.OrdinalIgnoreCase))
+                    .Select(h => h.Value)
+                    .SelectMany(v => v.Split(';'))
+                    .Select(c =>
+                    {
+                        var parts = c.Split('=', 2);
+                        return new HarCookie { Name = parts[0].Trim(), Value = parts.Length > 1 ? parts[1].Trim() : "" };
+                    })],
+                HeadersSize = request.Headers?.ToString()?.Length ?? 0,
+                BodySize = request.HasBody ? (request.BodyString?.Length ?? 0) : 0,
+                PostData = request.HasBody ? new HarPostData
+                {
+                    MimeType = request.ContentType,
+                    Text = request.BodyString ?? ""
+                }
+                    : null
+            },
+            Response = response is not null ? new HarResponse
+            {
+                Status = response.StatusCode,
+                StatusText = response.StatusDescription,
+                HttpVersion = $"HTTP/{response.HttpVersion}",
+                Headers = [.. response.Headers.Select(h => new HarHeader { Name = h.Name, Value = GetHeaderValue(h.Name, string.Join(", ", h.Value)) })],
+                Cookies = [.. response.Headers
+                    .Where(h => string.Equals(h.Name, "Set-Cookie", StringComparison.OrdinalIgnoreCase))
+                    .Select(h => h.Value)
+                    .Select(sc =>
+                    {
+                        var parts = sc.Split(';')[0].Split('=', 2);
+                        return new HarCookie { Name = parts[0].Trim(), Value = parts.Length > 1 ? parts[1].Trim() : "" };
+                    })],
+                Content = new HarContent
+                {
+                    Size = response.HasBody ? (response.BodyString?.Length ?? 0) : 0,
+                    MimeType = response.ContentType ?? "",
+                    Text = Configuration.IncludeResponse && response.HasBody ? response.BodyString : null
+                },
+                HeadersSize = response.Headers?.ToString()?.Length ?? 0,
+                BodySize = response.HasBody ? (response.BodyString?.Length ?? 0) : 0
+            } : null
+        };
+
+        return entry;
+    }
+}

DevProxy.Plugins/Generation/OpenApiSpecGeneratorPlugin.cs

@@ -288,13 +288,13 @@ private void SetParametersFromRequestHeaders(OpenApiOperation operation, HeaderC
         foreach (var header in headers)
         {
             var lowerCaseHeaderName = header.Name.ToLowerInvariant();
-            if (Http.StandardHeaders.Contains(lowerCaseHeaderName))
+            if (Models.Http.StandardHeaders.Contains(lowerCaseHeaderName))
             {
                 Logger.LogDebug("    Skipping standard header {HeaderName}", header.Name);
                 continue;
             }
 
-            if (Http.AuthHeaders.Contains(lowerCaseHeaderName))
+            if (Models.Http.AuthHeaders.Contains(lowerCaseHeaderName))
             {
                 Logger.LogDebug("    Skipping auth header {HeaderName}", header.Name);
                 continue;
@@ -388,13 +388,13 @@ private void SetResponseFromSession(OpenApiOperation operation, Response respons
             foreach (var header in response.Headers)
             {
                 var lowerCaseHeaderName = header.Name.ToLowerInvariant();
-                if (Http.StandardHeaders.Contains(lowerCaseHeaderName))
+                if (Models.Http.StandardHeaders.Contains(lowerCaseHeaderName))
                 {
                     Logger.LogDebug("    Skipping standard header {HeaderName}", header.Name);
                     continue;
                 }
 
-                if (Http.AuthHeaders.Contains(lowerCaseHeaderName))
+                if (Models.Http.AuthHeaders.Contains(lowerCaseHeaderName))
                 {
                     Logger.LogDebug("    Skipping auth header {HeaderName}", header.Name);
                     continue;

DevProxy.Plugins/Generation/TypeSpecGeneratorPlugin.cs

@@ -175,7 +175,7 @@ private void ProcessAuth(Request httpRequest, TypeSpecFile doc, Operation op)
         Logger.LogTrace("Entered {Name}", nameof(ProcessAuth));
 
         var authHeaders = httpRequest.Headers
-            .Where(h => Http.AuthHeaders.Contains(h.Name.ToLowerInvariant()))
+            .Where(h => Models.Http.AuthHeaders.Contains(h.Name.ToLowerInvariant()))
             .Select(h => (h.Name, h.Value));
 
         foreach (var (name, value) in authHeaders)
@@ -199,7 +199,7 @@ private void ProcessAuth(Request httpRequest, TypeSpecFile doc, Operation op)
 
         var query = HttpUtility.ParseQueryString(httpRequest.RequestUri.Query);
         var authQueryParam = query.AllKeys
-            .FirstOrDefault(k => k is not null && Http.AuthHeaders.Contains(k.ToLowerInvariant()));
+            .FirstOrDefault(k => k is not null && Models.Http.AuthHeaders.Contains(k.ToLowerInvariant()));
         if (authQueryParam is not null)
         {
             Logger.LogDebug("Found auth query parameter: {AuthQueryParam}", authQueryParam);
@@ -357,8 +357,8 @@ private void ProcessRequestHeaders(Request httpRequest, Operation op)
 
         foreach (var header in httpRequest.Headers)
         {
-            if (Http.StandardHeaders.Contains(header.Name.ToLowerInvariant()) ||
-                Http.AuthHeaders.Contains(header.Name.ToLowerInvariant()))
+            if (Models.Http.StandardHeaders.Contains(header.Name.ToLowerInvariant()) ||
+                Models.Http.AuthHeaders.Contains(header.Name.ToLowerInvariant()))
             {
                 continue;
             }
@@ -400,8 +400,8 @@ private async Task ProcessResponseAsync(Response? httpResponse, TypeSpecFile doc
             {
                 StatusCode = httpResponse.StatusCode,
                 Headers = httpResponse.Headers
-                    .Where(h => !Http.StandardHeaders.Contains(h.Name.ToLowerInvariant()) &&
-                                !Http.AuthHeaders.Contains(h.Name.ToLowerInvariant()))
+                    .Where(h => !Models.Http.StandardHeaders.Contains(h.Name.ToLowerInvariant()) &&
+                                !Models.Http.AuthHeaders.Contains(h.Name.ToLowerInvariant()))
                     .ToDictionary(h => h.Name.ToCamelCase(), h => h.Value.GetType().Name)
             };
 
@@ -688,7 +688,7 @@ private bool IsParametrizable(string segment)
             var query = HttpUtility.ParseQueryString(url.Query);
             foreach (string key in query.Keys)
             {
-                if (Http.AuthHeaders.Contains(key.ToLowerInvariant()))
+                if (Models.Http.AuthHeaders.Contains(key.ToLowerInvariant()))
                 {
                     Logger.LogDebug("Skipping auth header: {Key}", key);
                     continue;

DevProxy.Plugins/Models/Har.cs

@@ -0,0 +1,117 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+namespace DevProxy.Plugins.Models;
+
+internal sealed class HarFile
+{
+    public HarLog? Log { get; set; }
+}
+
+internal sealed class HarLog
+{
+    public string Version { get; set; } = "1.2";
+    public HarCreator Creator { get; set; } = new();
+    public List<HarEntry> Entries { get; set; } = [];
+}
+
+internal sealed class HarCreator
+{
+    public string? Name { get; set; }
+    public string? Version { get; set; }
+}
+
+internal sealed class HarEntry
+{
+    public string? StartedDateTime { get; set; }
+    public double Time { get; set; }
+    public HarRequest? Request { get; set; }
+    public HarResponse? Response { get; set; }
+    public HarCache Cache { get; set; } = new();
+    public HarTimings Timings { get; set; } = new();
+}
+
+internal sealed class HarRequest
+{
+    public string? Method { get; set; }
+    public string? Url { get; set; }
+    public string? HttpVersion { get; set; }
+    public List<HarHeader> Headers { get; set; } = [];
+    public List<HarQueryParam> QueryString { get; set; } = [];
+    public List<HarCookie> Cookies { get; set; } = [];
+    public long HeadersSize { get; set; }
+    public long BodySize { get; set; }
+    public HarPostData? PostData { get; set; }
+}
+
+internal sealed class HarResponse
+{
+    public int Status { get; set; }
+    public string? StatusText { get; set; }
+    public string? HttpVersion { get; set; }
+    public List<HarHeader> Headers { get; set; } = [];
+    public List<HarCookie> Cookies { get; set; } = [];
+    public HarContent Content { get; set; } = new();
+    public string RedirectURL { get; set; } = "";
+    public long HeadersSize { get; set; }
+    public long BodySize { get; set; }
+}
+
+internal sealed class HarHeader
+{
+    public string? Name { get; set; }
+    public string? Value { get; set; }
+}
+
+internal sealed class HarQueryParam
+{
+    public string? Name { get; set; }
+    public string? Value { get; set; }
+}
+
+internal sealed class HarCookie
+{
+    public string? Name { get; set; }
+    public string? Value { get; set; }
+    public string? Path { get; set; }
+    public string? Domain { get; set; }
+    public string? Expires { get; set; }
+    public bool? HttpOnly { get; set; }
+    public bool? Secure { get; set; }
+}
+
+internal sealed class HarPostData
+{
+    public string? MimeType { get; set; }
+    public string? Text { get; set; }
+    public List<HarParam>? Params { get; set; }
+}
+
+internal sealed class HarParam
+{
+    public string? Name { get; set; }
+    public string? Value { get; set; }
+    public string? FileName { get; set; }
+    public string? ContentType { get; set; }
+}
+
+internal sealed class HarContent
+{
+    public long Size { get; set; }
+    public string MimeType { get; set; } = "";
+    public string? Text { get; set; }
+    public string? Encoding { get; set; }
+}
+
+internal sealed class HarCache
+{
+    // Minimal - can be expanded if needed
+}
+
+internal sealed class HarTimings
+{
+    public double Send { get; set; }
+    public double Wait { get; set; }
+    public double Receive { get; set; }
+}

DevProxy.Plugins/Models/Http.cs

@@ -2,6 +2,8 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
+namespace DevProxy.Plugins.Models;
+
 internal static class Http
 {
     // from: https://github.com/jonluca/har-to-openapi/blob/0d44409162c0a127cdaccd60b0a270ecd361b829/src/utils/headers.ts
@@ -237,4 +239,22 @@ internal static class Http
         "apikey",
         "code"
     ];
+
+    internal static readonly string[] SensitiveHeaders =
+    [
+        "authorization",
+        "cookie",
+        "from",
+        "proxy-authenticate",
+        "proxy-authorization",
+        "set-cookie",
+        "www-authenticate",
+        "x-api-key",
+        "x-auth-token",
+        "x-csrf-token",
+        "x-forwarded-for",
+        "x-real-ip",
+        "x-session-token",
+        "x-xsrf-token"
+    ];
 }