From 9946efe7988241e24cba6116c9463c010c89707b Mon Sep 17 00:00:00 2001 From: Timo Salola Date: Fri, 4 Mar 2022 23:55:12 +0200 Subject: [PATCH 1/5] Support custom key mapping --- ...ltSecretManagerConfigurationLoaderTests.cs | 40 +++++++++++++++++++ ...SecretManagerConfigurationProviderTests.cs | 6 +-- .../ISecretManagerConfigurationLoader.cs | 10 +++++ ...DefaultSecretManagerConfigurationLoader.cs | 15 +++++++ .../SecretManagerConfigurationOptions.cs | 5 ++- .../SecretManagerConfigurationProvider.cs | 29 +++++++++----- .../SecretManagerConfigurationSource.cs | 3 +- 7 files changed, 92 insertions(+), 16 deletions(-) create mode 100644 Gcp.SecretManager.Provider.Tests/DefaultSecretManagerConfigurationLoaderTests.cs create mode 100644 Gcp.SecretManager.Provider/Contracts/ISecretManagerConfigurationLoader.cs create mode 100644 Gcp.SecretManager.Provider/DefaultSecretManagerConfigurationLoader.cs diff --git a/Gcp.SecretManager.Provider.Tests/DefaultSecretManagerConfigurationLoaderTests.cs b/Gcp.SecretManager.Provider.Tests/DefaultSecretManagerConfigurationLoaderTests.cs new file mode 100644 index 0000000..1216af6 --- /dev/null +++ b/Gcp.SecretManager.Provider.Tests/DefaultSecretManagerConfigurationLoaderTests.cs @@ -0,0 +1,40 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Google.Cloud.SecretManager.V1; +using Xunit; + +namespace Gcp.SecretManager.Provider.Tests +{ + public class DefaultSecretManagerConfigurationLoaderTests + { + + private readonly DefaultSecretManagerConfigurationLoader _target; + public DefaultSecretManagerConfigurationLoaderTests() + { + _target = new DefaultSecretManagerConfigurationLoader(); + } + + [Fact] + public void Load_ReturnsAlwaysTrue() + { + var result = _target.Load(null); + + Assert.True(result); + } + + [Theory] + [InlineData("This__Is__MultiLevel", "This:Is:MultiLevel")] + [InlineData("SingleLevel", "SingleLevel")] + [InlineData("MultiLevel_With__One_UnderScore", "MultiLevel_With:One_UnderScore")] + public void GetKey_ReplacesUnderscoresWithColon(string secret, string expected) + { + var result = _target.GetKey(new Secret + { + SecretName = new SecretName("SomeProjectId", secret) + }); + + Assert.Equal(expected, result); + } + } +} diff --git a/Gcp.SecretManager.Provider.Tests/SecretManagerConfigurationProviderTests.cs b/Gcp.SecretManager.Provider.Tests/SecretManagerConfigurationProviderTests.cs index 01dde66..e32d3b6 100644 --- a/Gcp.SecretManager.Provider.Tests/SecretManagerConfigurationProviderTests.cs +++ b/Gcp.SecretManager.Provider.Tests/SecretManagerConfigurationProviderTests.cs @@ -64,7 +64,7 @@ public SecretManagerConfigurationProviderTests() [Fact] public void Should_FetchSecrets_When_LoadIsCalled() { - var configurationProvider = new SecretManagerConfigurationProvider(_mockClient.Object, new ProjectName(_projectName)); + var configurationProvider = new SecretManagerConfigurationProvider(_mockClient.Object, new ProjectName(_projectName), new DefaultSecretManagerConfigurationLoader()); configurationProvider.Load(); foreach (var secret in _testSecrets) @@ -90,7 +90,7 @@ public void Should_FetchSecrets_When_LoadIsCalled_And_ExceptionIsThrown() svn.SecretVersionId == "latest"), null)) .ThrowsAsync(new Grpc.Core.RpcException(Grpc.Core.Status.DefaultCancelled)); - var configurationProvider = new SecretManagerConfigurationProvider(_mockClient.Object, new ProjectName(_projectName)); + var configurationProvider = new SecretManagerConfigurationProvider(_mockClient.Object, new ProjectName(_projectName), new DefaultSecretManagerConfigurationLoader()); configurationProvider.Load(); foreach (var secret in _testSecrets.Where(x => x.SecretName.SecretId != errorSecretId)) @@ -129,7 +129,7 @@ public void Should_FetcHierarchicalSecrets_When_LoadIsCalled() _mockClient.Setup(x => x.ListSecrets(It.Is(pn => pn.ProjectId == _projectName), null, null, null)) .Returns(pagedResponse); - var configurationProvider = new SecretManagerConfigurationProvider(_mockClient.Object, new ProjectName(_projectName)); + var configurationProvider = new SecretManagerConfigurationProvider(_mockClient.Object, new ProjectName(_projectName), new DefaultSecretManagerConfigurationLoader()); configurationProvider.Load(); Assert.True(configurationProvider.TryGet(dotNetName, out var configValue)); diff --git a/Gcp.SecretManager.Provider/Contracts/ISecretManagerConfigurationLoader.cs b/Gcp.SecretManager.Provider/Contracts/ISecretManagerConfigurationLoader.cs new file mode 100644 index 0000000..222ee79 --- /dev/null +++ b/Gcp.SecretManager.Provider/Contracts/ISecretManagerConfigurationLoader.cs @@ -0,0 +1,10 @@ +using Google.Cloud.SecretManager.V1; + +namespace Gcp.SecretManager.Provider.Contracts +{ + public interface ISecretManagerConfigurationLoader + { + string GetKey(Secret secret); + bool Load(Secret secret); + } +} diff --git a/Gcp.SecretManager.Provider/DefaultSecretManagerConfigurationLoader.cs b/Gcp.SecretManager.Provider/DefaultSecretManagerConfigurationLoader.cs new file mode 100644 index 0000000..b93f1bb --- /dev/null +++ b/Gcp.SecretManager.Provider/DefaultSecretManagerConfigurationLoader.cs @@ -0,0 +1,15 @@ +using Gcp.SecretManager.Provider.Contracts; +using Google.Cloud.SecretManager.V1; +using Microsoft.Extensions.Configuration; + +namespace Gcp.SecretManager.Provider +{ + public class DefaultSecretManagerConfigurationLoader : ISecretManagerConfigurationLoader + { + public string GetKey(Secret secret) + => secret.SecretName.SecretId.Replace("__", ConfigurationPath.KeyDelimiter); + + public bool Load(Secret secret) + => true; + } +} diff --git a/Gcp.SecretManager.Provider/SecretManagerConfigurationOptions.cs b/Gcp.SecretManager.Provider/SecretManagerConfigurationOptions.cs index e7e599f..a175543 100644 --- a/Gcp.SecretManager.Provider/SecretManagerConfigurationOptions.cs +++ b/Gcp.SecretManager.Provider/SecretManagerConfigurationOptions.cs @@ -1,8 +1,11 @@ -namespace Gcp.SecretManager.Provider +using Gcp.SecretManager.Provider.Contracts; + +namespace Gcp.SecretManager.Provider { public class SecretManagerConfigurationOptions { public string CredentialsPath { get; set; } public string ProjectId { get; set; } + public ISecretManagerConfigurationLoader Loader { get; set; } } } diff --git a/Gcp.SecretManager.Provider/SecretManagerConfigurationProvider.cs b/Gcp.SecretManager.Provider/SecretManagerConfigurationProvider.cs index bcb56d0..631f8c7 100644 --- a/Gcp.SecretManager.Provider/SecretManagerConfigurationProvider.cs +++ b/Gcp.SecretManager.Provider/SecretManagerConfigurationProvider.cs @@ -2,18 +2,24 @@ using Google.Cloud.SecretManager.V1; using Microsoft.Extensions.Configuration; using System.Threading.Tasks; +using Gcp.SecretManager.Provider.Contracts; namespace Gcp.SecretManager.Provider { public class SecretManagerConfigurationProvider : ConfigurationProvider { private readonly SecretManagerServiceClient _client; - private ProjectName _projectName; + private readonly ProjectName _projectName; + private readonly ISecretManagerConfigurationLoader _loader; - public SecretManagerConfigurationProvider(SecretManagerServiceClient client, ProjectName projectName) + public SecretManagerConfigurationProvider( + SecretManagerServiceClient client, + ProjectName projectName, + ISecretManagerConfigurationLoader loader) { _client = client; _projectName = projectName; + _loader = loader; } public override void Load() @@ -29,21 +35,22 @@ public async Task LoadAsync() { try { - var secretVersionName = new SecretVersionName(secret.SecretName.ProjectId, secret.SecretName.SecretId, "latest"); + if (!_loader.Load(secret)) + { + continue; + } + + var secretVersionName = new SecretVersionName(secret.SecretName.ProjectId, + secret.SecretName.SecretId, "latest"); var secretVersion = await _client.AccessSecretVersionAsync(secretVersionName); - Set(ConvertDelimiter(secret.SecretName.SecretId), secretVersion.Payload.Data.ToStringUtf8()); - } catch (Grpc.Core.RpcException) + Set(_loader.GetKey(secret), secretVersion.Payload.Data.ToStringUtf8()); + } + catch (Grpc.Core.RpcException) { // This might happen if secret is created but it has no versions available // For now just ignore. Maybe in future we should log that something went wrong? } } - - } - - private static string ConvertDelimiter(string key) - { - return key.Replace("__", ConfigurationPath.KeyDelimiter); } } } diff --git a/Gcp.SecretManager.Provider/SecretManagerConfigurationSource.cs b/Gcp.SecretManager.Provider/SecretManagerConfigurationSource.cs index 65183c1..b5ff82a 100644 --- a/Gcp.SecretManager.Provider/SecretManagerConfigurationSource.cs +++ b/Gcp.SecretManager.Provider/SecretManagerConfigurationSource.cs @@ -31,8 +31,9 @@ public IConfigurationProvider Build(IConfigurationBuilder builder) var projectName = new ProjectName(_options.ProjectId); var client = CreateClient(); + var loader = _options.Loader ?? new DefaultSecretManagerConfigurationLoader(); - return new SecretManagerConfigurationProvider(client, projectName); + return new SecretManagerConfigurationProvider(client, projectName, loader); } private SecretManagerServiceClient CreateClient() From a260bd09e7e2495de53dbd0b9bda72e40b80ef11 Mon Sep 17 00:00:00 2001 From: Timo Salola Date: Sat, 5 Mar 2022 00:05:09 +0200 Subject: [PATCH 2/5] Update README.md --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 36c24c7..086718c 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,16 @@ dotnet add package Gcp.SecretManager.Provider config.AddGcpSecretManager(options => { options.ProjectId = "ProjectId"; // Required options.CredentialsPath = "/path/to/credentials"; // Optional + options.Loader = new DefaultSecretManagerConfigurationLoader() // Optional, see more info below }); ``` *You can also provide CredentialsPath with GOOGLE_APPLICATION_CREDENTIALS environment variable* -3. Ready to go! \ No newline at end of file +3. Ready to go! + +## Loaders +Loaders handles if secret should be loaded and mapping from Secret Manager keys to application configuration values by implementing contract `ISecretManagerConfigurationLoader`. This can be passed as an option during setup. + +Contract exposes two method: `Load` and `GetKey`. `Load` method determines if the key should be loaded or not and `GetKey` handles mapping from secret to application configuration. You may access secret ID from `secret.SecretName.SecretId` + +If no loader is specified then `DefaultSecretManagerConfigurationLoader` will be used. It loads all keys and hierarcy is added by adding two underscores in the secret name. Eg. `MultiLevel__Secret` maps to `MultiLevel:Secret` key in application configuration. \ No newline at end of file From 5b5847ea16328b0a5045999dfb136c3bde6450f0 Mon Sep 17 00:00:00 2001 From: Timo Salola Date: Sat, 5 Mar 2022 00:05:31 +0200 Subject: [PATCH 3/5] Make default loader methods virtual so they can be overridden --- .../DefaultSecretManagerConfigurationLoader.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gcp.SecretManager.Provider/DefaultSecretManagerConfigurationLoader.cs b/Gcp.SecretManager.Provider/DefaultSecretManagerConfigurationLoader.cs index b93f1bb..fccb7c5 100644 --- a/Gcp.SecretManager.Provider/DefaultSecretManagerConfigurationLoader.cs +++ b/Gcp.SecretManager.Provider/DefaultSecretManagerConfigurationLoader.cs @@ -6,10 +6,10 @@ namespace Gcp.SecretManager.Provider { public class DefaultSecretManagerConfigurationLoader : ISecretManagerConfigurationLoader { - public string GetKey(Secret secret) + public virtual string GetKey(Secret secret) => secret.SecretName.SecretId.Replace("__", ConfigurationPath.KeyDelimiter); - public bool Load(Secret secret) + public virtual bool Load(Secret secret) => true; } } From 6e8c82f1ffddee1a80927c8444e2ad692aad54d0 Mon Sep 17 00:00:00 2001 From: Timo Salola Date: Sat, 5 Mar 2022 00:11:14 +0200 Subject: [PATCH 4/5] Small refactor for tests --- ...SecretManagerConfigurationProviderTests.cs | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Gcp.SecretManager.Provider.Tests/SecretManagerConfigurationProviderTests.cs b/Gcp.SecretManager.Provider.Tests/SecretManagerConfigurationProviderTests.cs index e32d3b6..55feb67 100644 --- a/Gcp.SecretManager.Provider.Tests/SecretManagerConfigurationProviderTests.cs +++ b/Gcp.SecretManager.Provider.Tests/SecretManagerConfigurationProviderTests.cs @@ -16,6 +16,8 @@ public class SecretManagerConfigurationProviderTests private const string _projectName = "TestProjectName"; private const string _secretProjectName = "TestSecretProjectName"; + private readonly SecretManagerConfigurationProvider _target; + public SecretManagerConfigurationProviderTests() { _testSecrets = new List @@ -59,17 +61,18 @@ public SecretManagerConfigurationProviderTests() svn.SecretVersionId == "latest"), null)) .ReturnsAsync(response); } + + _target = new SecretManagerConfigurationProvider(_mockClient.Object, new ProjectName(_projectName), new DefaultSecretManagerConfigurationLoader()); } [Fact] public void Should_FetchSecrets_When_LoadIsCalled() { - var configurationProvider = new SecretManagerConfigurationProvider(_mockClient.Object, new ProjectName(_projectName), new DefaultSecretManagerConfigurationLoader()); - configurationProvider.Load(); + _target.Load(); foreach (var secret in _testSecrets) { - configurationProvider.TryGet(secret.SecretName.SecretId, out string value); + _target.TryGet(secret.SecretName.SecretId, out string value); Assert.Equal($"{secret.SecretName.SecretId}-Value", value); } } @@ -90,12 +93,11 @@ public void Should_FetchSecrets_When_LoadIsCalled_And_ExceptionIsThrown() svn.SecretVersionId == "latest"), null)) .ThrowsAsync(new Grpc.Core.RpcException(Grpc.Core.Status.DefaultCancelled)); - var configurationProvider = new SecretManagerConfigurationProvider(_mockClient.Object, new ProjectName(_projectName), new DefaultSecretManagerConfigurationLoader()); - configurationProvider.Load(); + _target.Load(); foreach (var secret in _testSecrets.Where(x => x.SecretName.SecretId != errorSecretId)) { - configurationProvider.TryGet(secret.SecretName.SecretId, out string value); + _target.TryGet(secret.SecretName.SecretId, out string value); Assert.Equal($"{secret.SecretName.SecretId}-Value", value); } } @@ -129,10 +131,10 @@ public void Should_FetcHierarchicalSecrets_When_LoadIsCalled() _mockClient.Setup(x => x.ListSecrets(It.Is(pn => pn.ProjectId == _projectName), null, null, null)) .Returns(pagedResponse); - var configurationProvider = new SecretManagerConfigurationProvider(_mockClient.Object, new ProjectName(_projectName), new DefaultSecretManagerConfigurationLoader()); - configurationProvider.Load(); - Assert.True(configurationProvider.TryGet(dotNetName, out var configValue)); + _target.Load(); + + Assert.True(_target.TryGet(dotNetName, out var configValue)); Assert.Equal(value, configValue); } } From 2bc2f5110ab51624f434b40e1fd5d7efeabb7d42 Mon Sep 17 00:00:00 2001 From: Timo Salola Date: Sat, 5 Mar 2022 00:22:22 +0200 Subject: [PATCH 5/5] Add tests to verify loader is called from provider --- ...SecretManagerConfigurationProviderTests.cs | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/Gcp.SecretManager.Provider.Tests/SecretManagerConfigurationProviderTests.cs b/Gcp.SecretManager.Provider.Tests/SecretManagerConfigurationProviderTests.cs index 55feb67..5f3150b 100644 --- a/Gcp.SecretManager.Provider.Tests/SecretManagerConfigurationProviderTests.cs +++ b/Gcp.SecretManager.Provider.Tests/SecretManagerConfigurationProviderTests.cs @@ -4,6 +4,8 @@ using Moq; using System.Collections.Generic; using System.Linq; +using Gcp.SecretManager.Provider.Contracts; +using Newtonsoft.Json.Bson; using Xunit; namespace Gcp.SecretManager.Provider.Tests @@ -137,5 +139,39 @@ public void Should_FetcHierarchicalSecrets_When_LoadIsCalled() Assert.True(_target.TryGet(dotNetName, out var configValue)); Assert.Equal(value, configValue); } + + [Fact] + public void Should_CallLoaderGetKey() + { + var mockLoader = new Mock(); + mockLoader.Setup(m => m.GetKey(It.IsAny())) + .Returns((Secret secret) => secret.SecretName.SecretId); + mockLoader.Setup(m => m.Load(It.IsAny())).Returns(true); + + var target = new SecretManagerConfigurationProvider(_mockClient.Object, new ProjectName(_projectName), mockLoader.Object); + + target.Load(); + + mockLoader.Verify(m => m.GetKey(It.IsAny()), Times.Exactly(_testSecrets.Count())); + } + + [Fact] + public void Should_NotLoadAllSecrets() + { + var mockLoader = new Mock(); + mockLoader.Setup(m => m.GetKey(It.IsAny())) + .Returns((Secret secret) => secret.SecretName.SecretId); + mockLoader.Setup(m => m.Load(It.IsAny())).Returns((Secret secret) => + secret.SecretName.SecretId != _testSecrets.First().SecretName.SecretId); + + var target = new SecretManagerConfigurationProvider(_mockClient.Object, new ProjectName(_projectName), mockLoader.Object); + + target.Load(); + + Assert.False(target.TryGet("SecretId1", out var configValue1)); + Assert.True(target.TryGet("SecretId2", out var configValue2)); + Assert.True(target.TryGet("SecretId3", out var configValue3)); + Assert.True(target.TryGet("SecretId4", out var configValue4)); + } } }