400511 secert key event handler

Defra.Cdp.Backend.Api.Tests/Defra.Cdp.Backend.Api.Tests.csproj

@@ -32,6 +32,9 @@
     <Content Include="Resources\testlayer.tgz">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="Resources\payload-get-all-secrets.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
   </ItemGroup>
 
   <ItemGroup>

Defra.Cdp.Backend.Api.Tests/Resources/payload-get-all-secrets.json

@@ -0,0 +1,20 @@
+{
+    "source": "cdp-secret-manager-lambda",
+    "statusCode": 200,
+    "action": "get_all_secret_keys",
+    "body": {
+        "get_all_secret_keys": true,
+        "keys": {
+            "cdp/services/test-service-one": [
+              "placeholder",
+              "EXAMPLE_ONE"
+            ],
+            "cdp/services/test-service-two": [
+              "placeholder",
+              "EXAMPLE_TWO"
+            ]
+        },
+        "exception": "",
+        "environment": "dev"
+    }
+}

Defra.Cdp.Backend.Api.Tests/Services/Secrets/SecretEventHandlerTest.cs

@@ -0,0 +1,71 @@
+using System.Text.Json;
+using Defra.Cdp.Backend.Api.Models;
+using Defra.Cdp.Backend.Api.Services.Secrets;
+using Defra.Cdp.Backend.Api.Services.Secrets.events;
+using NSubstitute;
+
+namespace Defra.Cdp.Backend.Api.Tests.Services.Secrets;
+
+public class SecretEventHandlerTest
+{
+
+    [Fact]
+    public async void WillProcessGetAllSecretsPayload()
+    {
+        var service =  Substitute.For<ISecretsService>();
+        var eventHandler = new SecretEventHandler(service, ConsoleLogger.CreateLogger<SecretEventHandler>());
+
+        var mockPayload = SecretEventHandler.TryParseMessageHeader(await File.ReadAllTextAsync("Resources/payload-get-all-secrets.json"));
+
+        Assert.NotNull(mockPayload);
+        service
+            .UpdateSecrets(Arg.Any<List<TenantSecrets>>(), Arg.Any<CancellationToken>())
+            .Returns(Task.CompletedTask);
+        await eventHandler.Handle(mockPayload, new CancellationToken());
+
+        await service.ReceivedWithAnyArgs().UpdateSecrets(Arg.Any<List<TenantSecrets>>(), Arg.Any<CancellationToken>());
+
+    }
+
+    [Fact]
+    public void TryParseMessageHeaderWithValidPayload()
+    {
+       var body = "{\"source\": \"cdp-secret-manager-lambda\", \"statusCode\": 200, \"action\": \"get_all_secret_keys\", \"body\": {}}";
+       var res = SecretEventHandler.TryParseMessageHeader(body);
+       Assert.NotNull(res);
+    }
+
+    [Fact]
+    public void TryParseMessageHeaderInvalid()
+    {
+        var otherLambda = "{\"source\": \"cdp-some-other-lambda\", \"statusCode\": 200, \"action\": \"get_all_secret_keys\", \"body\": {}}";
+        var res = SecretEventHandler.TryParseMessageHeader(otherLambda);
+        Assert.Null(res);
+
+        var otherMessage = "{\"foo\": \"bar\"}";
+        res = SecretEventHandler.TryParseMessageHeader(otherMessage);
+        Assert.Null(res);
+
+        var invalidJson = "<tag>foo</tag>";
+        res = SecretEventHandler.TryParseMessageHeader(invalidJson);
+        Assert.Null(res);
+    }
+
+    [Fact]
+    public void CanExtractBody()
+    {
+        var body = "{\"source\": \"cdp-secret-manager-lambda\", \"statusCode\": 200, \"action\": \"get_all_secret_keys\", \"body\": " +
+                   "{ \"get_all_secret_keys\": true, " +
+                   "\"exception\": \"\", " +
+                   "\"environment\": \"dev\", " +
+                   "\"keys\": {\"cdp/service/foo\": [\"FOO\"]}" +
+                   "}}";
+        var res = SecretEventHandler.TryParseMessageHeader(body);
+        Assert.NotNull(res);
+
+        var parsedBody = res.Body.Deserialize<BodyGetAllSecretKeys>();
+        Assert.Equal("dev", parsedBody?.Environment);
+        Assert.Equal("", parsedBody?.Exception);
+        Assert.Equal(1, parsedBody?.Keys.Count);
+    }
+}

Defra.Cdp.Backend.Api/Config/SecretManagerListenerOptions.cs

@@ -0,0 +1,8 @@
+namespace Defra.Cdp.Backend.Api.Config;
+
+public class SecretEventListenerOptions
+{
+    public const string Prefix = "SecretManagerEvents";
+    public string QueueUrl { get; set; } = null!;
+    public bool Enabled { get; set; }
+}

Defra.Cdp.Backend.Api/Endpoints/DeploymentsEndpointV2.cs

@@ -1,5 +1,6 @@
 using Defra.Cdp.Backend.Api.Models;
 using Defra.Cdp.Backend.Api.Services.Deployments;
+using Defra.Cdp.Backend.Api.Services.Secrets;
 using FluentValidation;
 using Microsoft.AspNetCore.Mvc;
 
@@ -77,7 +78,7 @@ public static IEndpointRouteBuilder MapDeploymentsEndpointV2(this IEndpointRoute
         var deployment = await deploymentsService.FindDeployment(deploymentId, cancellationToken);
 
         return deployment == null
-            ? Results.NotFound(new { Message = $"{deploymentId} was not found" })
+            ? Results.NotFound(new ApiError($"{deploymentId} was not found"))
             : Results.Ok(deployment);
     }
 
@@ -99,17 +100,18 @@ public static IEndpointRouteBuilder MapDeploymentsEndpointV2(this IEndpointRoute
 
     private static async Task<IResult> RegisterDeployment(
         IDeploymentsServiceV2 deploymentsServiceV2,
+        ISecretsService secretsService,
         IValidator<RequestedDeployment> validator,
-        RequestedDeployment rd,
+        RequestedDeployment requestedDeployment,
         ILoggerFactory loggerFactory,
         CancellationToken cancellationToken)
     {
-        var validatedResult = await validator.ValidateAsync(rd, cancellationToken);
+        var validatedResult = await validator.ValidateAsync(requestedDeployment, cancellationToken);
         if (!validatedResult.IsValid) return Results.ValidationProblem(validatedResult.ToDictionary());
-
+        
         var logger = loggerFactory.CreateLogger("RegisterDeployment");
-        logger.LogInformation("Registering deployment {RdDeploymentId}", rd.DeploymentId);
-        await deploymentsServiceV2.RegisterDeployment(rd, cancellationToken);
+        logger.LogInformation("Registering deployment {DeploymentId}", requestedDeployment.DeploymentId);
+        await deploymentsServiceV2.RegisterDeployment(requestedDeployment, cancellationToken);
         return Results.Ok();
     }
 

Defra.Cdp.Backend.Api/Endpoints/TenantSecretsEndpoint.cs

@@ -0,0 +1,21 @@
+using Defra.Cdp.Backend.Api.Models;
+using Defra.Cdp.Backend.Api.Services.Secrets;
+using Microsoft.AspNetCore.Mvc;
+
+namespace Defra.Cdp.Backend.Api.Endpoints;
+
+public static class TenantSecretsEndpoint
+{
+    public static IEndpointRouteBuilder MapTenantSecretsEndpoint(this IEndpointRouteBuilder app)
+    {
+        app.MapGet("secrets/{environment}/{service}", FindTenantSecrets);
+        return app;
+    }
+
+    static async Task<IResult> FindTenantSecrets([FromServices] ISecretsService secretsService, string environment,
+        string service, CancellationToken cancellationToken)
+    {
+        var secrets = await secretsService.FindSecrets(environment, service, cancellationToken);
+        return secrets == null ? Results.NotFound(new ApiError("secrets not found")) : Results.Ok(secrets);
+    }
+}

Defra.Cdp.Backend.Api/Models/DeploymentV2.cs

@@ -31,6 +31,9 @@ public class DeploymentV2
     public string Status { get; set; }
     public bool Unstable { get; set; } = false;
 
+    public string? ConfigVersion { get; init; } = default!;
+    public List<string> Secrets { get; init; } = new();
+
     public static DeploymentV2 FromRequest(RequestedDeployment req)
     {
         return new DeploymentV2
@@ -46,6 +49,8 @@ public static DeploymentV2 FromRequest(RequestedDeployment req)
             Created = DateTime.Now,
             Updated = DateTime.Now,
             Status = req.InstanceCount > 0 ? Requested : Undeployed,
+            ConfigVersion = req.ConfigVersion,
+            Secrets = req.Secrets
         };
     }
 

Defra.Cdp.Backend.Api/Models/RequestedDeployment.cs

@@ -31,4 +31,11 @@ public sealed class RequestedDeployment
 
     [property: JsonPropertyName("deploymentId")]
     public string DeploymentId { get; init; } = default!;
+
+    [property: JsonPropertyName("configVersion")]
+    public string? ConfigVersion { get; init; } = default!;
+
+    [property: JsonPropertyName("secrets")]
+    public List<string> Secrets { get; init; } = new();
+
 }

Defra.Cdp.Backend.Api/Models/TenantSecrets.cs

@@ -0,0 +1,23 @@
+using System.Text.Json.Serialization;
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Attributes;
+using MongoDB.Bson.Serialization.IdGenerators;
+
+namespace Defra.Cdp.Backend.Api.Models;
+
+public class TenantSecrets
+{
+    [BsonId(IdGenerator = typeof(ObjectIdGenerator))]
+    [BsonIgnoreIfDefault]
+    [JsonIgnore(Condition = JsonIgnoreCondition.Always)]
+    public ObjectId? Id { get; init; } = default;
+
+    [property: JsonPropertyName("service")]
+    public string Service { get; init; } = default!;
+
+    [property: JsonPropertyName("environment")]
+    public string Environment { get; init; } = default!;
+
+    [property: JsonPropertyName("secrets")]
+    public List<string> Secrets { get; init; } = new();
+}

Defra.Cdp.Backend.Api/Program.cs

@@ -9,6 +9,7 @@
 using Defra.Cdp.Backend.Api.Services.Deployments;
 using Defra.Cdp.Backend.Api.Services.Github;
 using Defra.Cdp.Backend.Api.Services.Github.ScheduledTasks;
+using Defra.Cdp.Backend.Api.Services.Secrets;
 using Defra.Cdp.Backend.Api.Services.TenantArtifacts;
 using Defra.Cdp.Backend.Api.Services.TestSuites;
 using Defra.Cdp.Backend.Api.Utils;
@@ -71,6 +72,7 @@
 // Setup the services
 builder.Services.Configure<EcsEventListenerOptions>(builder.Configuration.GetSection(EcsEventListenerOptions.Prefix));
 builder.Services.Configure<EcrEventListenerOptions>(builder.Configuration.GetSection(EcrEventListenerOptions.Prefix));
+builder.Services.Configure<SecretEventListenerOptions>(builder.Configuration.GetSection(SecretEventListenerOptions.Prefix));
 builder.Services.Configure<DockerServiceOptions>(builder.Configuration.GetSection(DockerServiceOptions.Prefix));
 builder.Services.Configure<DeployablesClientOptions>(builder.Configuration.GetSection(DeployablesClientOptions.Prefix));
 builder.Services.AddScoped<IValidator<RequestedDeployment>, RequestedDeploymentValidator>();
@@ -131,8 +133,12 @@
 builder.Services.AddSingleton<TemplatesFromConfig>();
 builder.Services.AddSingleton<ITemplatesService, TemplatesService>();
 builder.Services.AddSingleton<ITestRunService, TestRunService>();
+
 builder.Services.AddSingleton<DeploymentEventHandlerV2>();
 builder.Services.AddSingleton<LambdaMessageHandlerV2>();
+builder.Services.AddSingleton<ISecretsService, SecretsService>();
+builder.Services.AddSingleton<ISecretEventHandler, SecretEventHandler>();
+builder.Services.AddSingleton<SecretEventListener>();
 builder.Services.AddSingleton<MongoLock>();
 
 // Validators
@@ -172,6 +178,7 @@
 app.MapLibrariesEndpoint();
 app.MapRepositoriesEndpoint();
 app.MapTestSuiteEndpoint();
+app.MapTenantSecretsEndpoint();
 app.MapAdminEndpoint();
 app.MapHealthChecks("/health");
 
@@ -188,6 +195,13 @@
 Task.Run(() =>
     ecrSqsEventListener?.ReadAsync(app.Lifetime
         .ApplicationStopping)); // do not await this, we want it to run in the background
+
+var secretEventListener = app.Services.GetService<SecretEventListener>();
+logger.Information("Starting Secret Event listener - reading secret update events from SQS");
+Task.Run(() =>
+    secretEventListener?.ReadAsync(app.Lifetime
+        .ApplicationStopping)); // do not await this, we want it to run in the background
+
 #pragma warning restore CS4014
 
 app.Run();

Defra.Cdp.Backend.Api/Services/Secrets/SecretEventHandler.cs

@@ -0,0 +1,89 @@
+using System.Text.Json;
+using Defra.Cdp.Backend.Api.Models;
+using Defra.Cdp.Backend.Api.Services.Secrets.events;
+
+namespace Defra.Cdp.Backend.Api.Services.Secrets;
+
+public interface ISecretEventHandler
+{
+   Task Handle(MessageHeader header, CancellationToken cancellationToken);
+}
+
+/**
+ * Handles specific payloads sent by the secret manager lambda.
+ * All messages have the same outer body detailing the source & action.
+ */
+public class SecretEventHandler : ISecretEventHandler
+{
+
+    private readonly ISecretsService _secretsService;
+    private readonly ILogger<SecretEventHandler> _logger;
+
+    public SecretEventHandler(ISecretsService secretsService, ILogger<SecretEventHandler> logger)
+    {
+        _logger = logger;
+        _secretsService = secretsService;
+    }
+
+    public async Task Handle(MessageHeader header, CancellationToken cancellationToken)
+    {
+        switch (header.Action)
+        {
+            case "get_all_secret_keys":
+                await HandleGetAllSecrets(header, cancellationToken);
+                break;
+            default:
+                _logger.LogDebug("Ignoring action: {Action} not handled", header.Action);
+                return;
+        }
+    }
+
+    /**
+     * Handler for get_all_secret_keys action. Contains a dict of all the services in an environment that have
+     * secret values set along with a list of the key/environment variable the secret is bound to,
+     * but NOT the actual secret itself.
+     */
+    public async Task HandleGetAllSecrets(MessageHeader header, CancellationToken cancellationToken)
+    {
+        var body = header.Body?.Deserialize<BodyGetAllSecretKeys>();
+        if (body == null)
+        {
+            _logger.LogInformation("Failed to parse body of 'get_all_secret_keys' message");   
+            return;
+        }
+
+        if (body.Exception != "")
+        {
+            _logger.LogError("get_all_secret_keys message contained exception {}", body.Exception);
+            return;
+        } 
+
+        _logger.LogInformation("Updating secrets in {Environment}", body.Environment);
+        var secrets = new List<TenantSecrets>();
+
+        foreach (var kv in body.Keys)
+        {
+            var service = kv.Key.Replace("cdp/services/", "");
+            secrets.Add(new TenantSecrets
+            {
+                Service = service, Environment = body.Environment, Secrets = kv.Value
+            });
+        }
+
+        await _secretsService.UpdateSecrets(secrets, cancellationToken);
+        _logger.LogInformation("Updated secrets for {Environment}", body.Environment);
+    }
+
+    public static MessageHeader? TryParseMessageHeader(string body)
+    {
+        try
+        {
+            var header = JsonSerializer.Deserialize<MessageHeader>(body);
+            return header?.Source != "cdp-secret-manager-lambda" ? null : header;
+        }
+        catch(Exception e)
+        {
+            return null;
+        }
+    }
+}