Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Coordination.Kubernetes] Change hosting extension to use options instead of setup #1172

Merged
merged 1 commit into from
Jan 5, 2023
Merged

[Coordination.Kubernetes] Change hosting extension to use options instead of setup #1172

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/cluster.bootstrap/examples/discovery/kubernetes/src/KubernetesCluster/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public static async Task Main(string[] args)
Roles = new[] { "cluster" },
SplitBrainResolver = new LeaseMajorityOption
{
LeaseImplementation = KubernetesLeaseOption.Instance
LeaseImplementation = new KubernetesLeaseOption()
}
});

Expand Down
103 changes: 80 additions & 23 deletions src/coordination/kubernetes/Akka.Coordination.KubernetesApi.Tests/AkkaHostingSpec.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
using Akka.Actor.Setup;
using Akka.Hosting;
using FluentAssertions;
using Humanizer;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Xunit;
Expand All @@ -30,51 +31,107 @@ public void HostingExtension1Test()
builder.WithKubernetesLease();

builder.Configuration.HasValue.Should().BeTrue();
builder.Configuration.Value.GetConfig("akka.coordination.lease.kubernetes")
.Should().NotBeNull();
var config = builder.Configuration.Value.GetConfig(KubernetesLease.ConfigPath);
config.Should().NotBeNull();
config = config.WithFallback(builder.Configuration.Value.GetConfig("akka.coordination.lease"));

var settings = KubernetesSettings.Create(config);
settings.ApiCaPath.Should().Be("/var/run/secrets/kubernetes.io/serviceaccount/ca.crt");
settings.ApiTokenPath.Should().Be("/var/run/secrets/kubernetes.io/serviceaccount/token");
settings.ApiServiceHostEnvName.Should().Be("KUBERNETES_SERVICE_HOST");
settings.ApiServicePortEnvName.Should().Be("KUBERNETES_SERVICE_PORT");
settings.Namespace.Should().BeNull();
settings.NamespacePath.Should().Be("/var/run/secrets/kubernetes.io/serviceaccount/namespace");
settings.ApiServiceRequestTimeout.Should().Be(2.Seconds());
settings.Secure.Should().BeTrue();
settings.BodyReadTimeout.Should().Be(1.Seconds());

var timeSettings = TimeoutSettings.Create(config);
timeSettings.HeartbeatInterval.Should().Be(12.Seconds());
timeSettings.HeartbeatTimeout.Should().Be(120.Seconds());
timeSettings.OperationTimeout.Should().Be(5.Seconds());
}

[Fact(DisplayName = "Hosting Action<Setup> extension should add Setup class and default hocon settings")]
[Fact(DisplayName = "Hosting Action<KubernetesLeaseOption> extension should override hocon settings")]
public void HostingExtension2Test()
{
var builder = new AkkaConfigurationBuilder(new ServiceCollection(), "test");

builder.WithKubernetesLease(lease =>
{
lease.Namespace = "underTest";
lease.ApiCaPath = "a";
lease.ApiTokenPath = "b";
lease.ApiServiceHostEnvName = "c";
lease.ApiServicePortEnvName = "d";
lease.Namespace = "e";
lease.NamespacePath = "f";
lease.ApiServiceRequestTimeout = 3.Seconds();
lease.SecureApiServer = false;
lease.HeartbeatInterval = 4.Seconds();
lease.HeartbeatTimeout = 10.Seconds();
lease.LeaseOperationTimeout = 4.Seconds();
});

builder.Configuration.HasValue.Should().BeTrue();
builder.Configuration.Value.GetConfig("akka.coordination.lease.kubernetes")
.Should().NotBeNull();
var setup = ExtractSetup(builder);
setup.Should().NotBeNull();
// !: null check above
setup!.Namespace.Should().Be("underTest");
var config = builder.Configuration.Value.GetConfig(KubernetesLease.ConfigPath);
config.Should().NotBeNull();
config = config.WithFallback(builder.Configuration.Value.GetConfig("akka.coordination.lease"));

var settings = KubernetesSettings.Create(config);
settings.ApiCaPath.Should().Be("a");
settings.ApiTokenPath.Should().Be("b");
settings.ApiServiceHostEnvName.Should().Be("c");
settings.ApiServicePortEnvName.Should().Be("d");
settings.Namespace.Should().Be("e");
settings.NamespacePath.Should().Be("f");
settings.ApiServiceRequestTimeout.Should().Be(3.Seconds());
settings.Secure.Should().BeFalse();
settings.BodyReadTimeout.Should().Be(1.5.Seconds());

var timeSettings = TimeoutSettings.Create(config);
timeSettings.HeartbeatInterval.Should().Be(4.Seconds());
timeSettings.HeartbeatTimeout.Should().Be(10.Seconds());
timeSettings.OperationTimeout.Should().Be(4.Seconds());
}

[Fact(DisplayName = "Hosting Setup extension should add Setup class and default hocon settings")]
[Fact(DisplayName = "Hosting Setup extension should override hocon settings")]
public void HostingExtension3Test()
{
var builder = new AkkaConfigurationBuilder(new ServiceCollection(), "test");

builder.WithKubernetesLease(new KubernetesLeaseSetup
builder.WithKubernetesLease(new KubernetesLeaseOption
{
Namespace = "underTest"
ApiCaPath = "a",
ApiTokenPath = "b",
ApiServiceHostEnvName = "c",
ApiServicePortEnvName = "d",
Namespace = "e",
NamespacePath = "f",
ApiServiceRequestTimeout = 3.Seconds(),
SecureApiServer = false,
HeartbeatInterval = 4.Seconds(),
HeartbeatTimeout = 10.Seconds(),
LeaseOperationTimeout = 4.Seconds()
});

builder.Configuration.HasValue.Should().BeTrue();
builder.Configuration.Value.GetConfig("akka.coordination.lease.kubernetes")
.Should().NotBeNull();
var setup = ExtractSetup(builder);
setup.Should().NotBeNull();
// !: null check above
setup!.Namespace.Should().Be("underTest");
}
var config = builder.Configuration.Value.GetConfig(KubernetesLease.ConfigPath);
config.Should().NotBeNull();
var settings = KubernetesSettings.Create(config);
settings.ApiCaPath.Should().Be("a");
settings.ApiTokenPath.Should().Be("b");
settings.ApiServiceHostEnvName.Should().Be("c");
settings.ApiServicePortEnvName.Should().Be("d");
settings.Namespace.Should().Be("e");
settings.NamespacePath.Should().Be("f");
settings.ApiServiceRequestTimeout.Should().Be(3.Seconds());
settings.Secure.Should().BeFalse();
settings.BodyReadTimeout.Should().Be(1.5.Seconds());

private KubernetesLeaseSetup? ExtractSetup(AkkaConfigurationBuilder builder)
{
return builder.Setups.FirstOrDefault(s => s is KubernetesLeaseSetup) as KubernetesLeaseSetup;
var timeSettings = TimeoutSettings.Create(config);
timeSettings.HeartbeatInterval.Should().Be(4.Seconds());
timeSettings.HeartbeatTimeout.Should().Be(10.Seconds());
timeSettings.OperationTimeout.Should().Be(4.Seconds());
}
}
}
6 changes: 3 additions & 3 deletions src/coordination/kubernetes/Akka.Coordination.KubernetesApi.Tests/KubernetesSettingsSpec.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ private static KubernetesSettings Conf(string? overrides)
.WithFallback(LeaseProvider.DefaultConfig())
: KubernetesLease.DefaultConfiguration
.WithFallback(LeaseProvider.DefaultConfig());
return KubernetesSettings.Create(config.GetConfig(KubernetesLease.ConfigPath), TimeoutSettings.Create(config.GetConfig("akka.coordination.lease")));
return KubernetesSettings.Create(config.GetConfig(KubernetesLease.ConfigPath).WithFallback(config.GetConfig("akka.coordination.lease")));
}

[Fact(DisplayName = "default request-timeout should be 2/5 of the lease-operation-timeout")]
Expand All @@ -51,15 +51,15 @@ public void ShouldAllowServerRequestTimeoutOverride()
}

[Fact(DisplayName =
"Kubernetes settings should not allow server request timeout greater than operation timeout")]
"Kubernetes settings should not allow service request timeout greater than operation timeout")]
public void InvalidServerRequestTimeout()
{
Assert.Throws<ConfigurationException>(() =>
{
Conf(@"
akka.coordination.lease.lease-operation-timeout=5s
akka.coordination.lease.kubernetes.api-service-request-timeout=6s");
}).Message.Should().Be("'api-service-request-timeout can not be less than 'lease-operation-timeout'");
}).Message.Should().Be("'api-service-request-timeout can not be greater than 'lease-operation-timeout'");
}

[Fact(DisplayName = "KubernetesSettings should contain default values")]
Expand Down
31 changes: 8 additions & 23 deletions src/coordination/kubernetes/Akka.Coordination.KubernetesApi/AkkaHostingExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,20 +13,6 @@ namespace Akka.Coordination.KubernetesApi
{
public static class AkkaHostingExtensions
{
/// <summary>
/// Adds Akka.Coordination.KubernetesApi <see cref="Lease"/> support to the <see cref="ActorSystem"/>.
/// Note that this only adds the lease plugin, you will still need to add the services that depends on
/// <see cref="Lease"/> to use this.
/// </summary>
/// <param name="builder">
/// The builder instance being configured.
/// </param>
/// <returns>
/// The same <see cref="AkkaConfigurationBuilder"/> instance originally passed in.
/// </returns>
public static AkkaConfigurationBuilder WithKubernetesLease(this AkkaConfigurationBuilder builder)
=> builder.WithKubernetesLease(null as KubernetesLeaseSetup);

/// <summary>
/// Adds Akka.Coordination.KubernetesApi <see cref="Lease"/> support to the <see cref="ActorSystem"/>.
/// Note that this only adds the lease plugin, you will still need to add the services that depends on
Expand All @@ -36,20 +22,20 @@ public static AkkaConfigurationBuilder WithKubernetesLease(this AkkaConfiguratio
/// The builder instance being configured.
/// </param>
/// <param name="configure">
/// An action that modifies an <see cref="KubernetesLeaseSetup"/> instance, used
/// An action that modifies an <see cref="KubernetesLeaseOption"/> instance, used
/// to configure Akka.Coordination.KubernetesApi.
/// </param>
/// <returns>
/// The same <see cref="AkkaConfigurationBuilder"/> instance originally passed in.
/// </returns>
public static AkkaConfigurationBuilder WithKubernetesLease(
this AkkaConfigurationBuilder builder,
Action<KubernetesLeaseSetup> configure)
Action<KubernetesLeaseOption> configure)
{
if (configure == null)
throw new ArgumentNullException(nameof(configure));

var setup = new KubernetesLeaseSetup();
var setup = new KubernetesLeaseOption();
configure(setup);
return WithKubernetesLease(builder, setup);
}
Expand All @@ -62,20 +48,19 @@ public static AkkaConfigurationBuilder WithKubernetesLease(this AkkaConfiguratio
/// <param name="builder">
/// The builder instance being configured.
/// </param>
/// <param name="setup">
/// The <see cref="KubernetesLeaseSetup"/> instance used to configure Akka.Discovery.Azure.
/// <param name="options">
/// The <see cref="KubernetesLeaseOption"/> instance used to configure Akka.Discovery.Azure.
/// </param>
/// <returns>
/// The same <see cref="AkkaConfigurationBuilder"/> instance originally passed in.
/// </returns>
public static AkkaConfigurationBuilder WithKubernetesLease(
this AkkaConfigurationBuilder builder,
KubernetesLeaseSetup? setup)
KubernetesLeaseOption? options = null)
{
options?.Apply(builder);
builder.AddHocon(KubernetesLease.DefaultConfiguration, HoconAddMode.Append);
if (setup != null)
builder.AddSetup(setup);

builder.AddHocon(LeaseProvider.DefaultConfig(), HoconAddMode.Append);
return builder;
}

Expand Down
6 changes: 3 additions & 3 deletions src/coordination/kubernetes/Akka.Coordination.KubernetesApi/KubernetesLease.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@ public static Config DefaultConfiguration

private static string TruncateTo63Characters(string name) => name.Length > 63 ? name.Substring(0, 63) : name;

private static readonly Regex Rx1 = new Regex("[_.]");
private static readonly Regex Rx2 = new Regex("[^-a-z0-9]");
private static readonly Regex Rx1 = new ("[_.]", RegexOptions.Compiled);
private static readonly Regex Rx2 = new ("[^-a-z0-9]", RegexOptions.Compiled);
private static string MakeDns1039Compatible(string name)
{
var normalized = name.Normalize(NormalizationForm.FormKD).ToLowerInvariant();
Expand All @@ -57,7 +57,7 @@ public KubernetesLease(ExtendedActorSystem system, AtomicBoolean leaseTaken, Lea
_settings = settings;

_log = Logging.GetLogger(system, GetType());
var kubernetesSettings = KubernetesSettings.Create(system, settings.TimeoutSettings);
var kubernetesSettings = KubernetesSettings.Create(system);

var setup = system.Settings.Setup.Get<KubernetesLeaseSetup>();
if (setup.HasValue)
Expand Down
53 changes: 45 additions & 8 deletions src/coordination/kubernetes/Akka.Coordination.KubernetesApi/KubernetesLeaseOption.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,9 @@
// -----------------------------------------------------------------------

using System;
using System.Text;
using Akka.Actor.Setup;
using Akka.Cluster.Hosting.SBR;
using Akka.Configuration;
using Akka.Hosting;
using Akka.Hosting.Coordination;

Expand All @@ -15,18 +16,54 @@ namespace Akka.Coordination.KubernetesApi
{
public class KubernetesLeaseOption: LeaseOptionBase
{
public static readonly KubernetesLeaseOption Instance = new ();

private KubernetesLeaseOption()
{
}
public string? ApiCaPath { get; set; }
public string? ApiTokenPath { get; set; }
public string? ApiServiceHostEnvName { get; set; }
public string? ApiServicePortEnvName { get; set; }
public string? Namespace { get; set; }
public string? NamespacePath { get; set; }
public TimeSpan? ApiServiceRequestTimeout { get; set; }
public bool? SecureApiServer { get; set; }
public TimeSpan? HeartbeatInterval { get; set; }
public TimeSpan? HeartbeatTimeout { get; set; }
public TimeSpan? LeaseOperationTimeout { get; set; }

public override string ConfigPath => KubernetesLease.ConfigPath;
public override Type Class { get; } = typeof(KubernetesLease);

public override void Apply(AkkaConfigurationBuilder builder, Setup? setup = null)
public override void Apply(AkkaConfigurationBuilder builder, Setup? inputSetup = null)
{
throw new NotImplementedException("Not intended to be applied, use the `WithKubernetesLease()` Akka.Hosting extension method instead.");
var sb = new StringBuilder();
sb.AppendLine($"{ConfigPath} {{");
sb.AppendLine($"lease-class = {Class.AssemblyQualifiedName!.ToHocon()}");
if (ApiCaPath is { })
sb.AppendLine($"api-ca-path = {ApiCaPath.ToHocon()}");
if (ApiTokenPath is { })
sb.AppendLine($"api-token-path = {ApiTokenPath.ToHocon()}");
if (ApiServiceHostEnvName is { })
sb.AppendLine($"api-service-host-env-name = {ApiServiceHostEnvName.ToHocon()}");
if (ApiServicePortEnvName is { })
sb.AppendLine($"api-service-port-env-name = {ApiServicePortEnvName.ToHocon()}");
if (NamespacePath is { })
sb.AppendLine($"namespace-path = {NamespacePath.ToHocon()}");
if (Namespace is { })
sb.AppendLine($"namespace = {Namespace.ToHocon()}");
if (ApiServiceRequestTimeout is { })
sb.AppendLine($"api-service-request-timeout = {ApiServiceRequestTimeout.ToHocon()}");
if (SecureApiServer is { })
sb.AppendLine($"secure-api-server = {SecureApiServer.ToHocon()}");
if (HeartbeatInterval is { })
sb.AppendLine($"heartbeat-interval = {HeartbeatInterval.ToHocon()}");
if (HeartbeatTimeout is { })
sb.AppendLine($"heartbeat-timeout = {HeartbeatTimeout.ToHocon()}");
if (LeaseOperationTimeout is { })
sb.AppendLine($"lease-operation-timeout = {LeaseOperationTimeout.ToHocon()}");
sb.AppendLine("}");

//var config = ConfigurationFactory.ParseString(sb.ToString())
// .WithFallback(LeaseProvider.DefaultConfig().GetConfig("akka.coordination.lease"));

builder.AddHocon(sb.ToString(), HoconAddMode.Prepend);
}
}
}