From 6be555515566bb4b2faa41062e5b6369aed4e809 Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Tue, 4 Mar 2025 16:57:30 -0800 Subject: [PATCH 01/26] expose command line parameter for endpoints --- libs/host/Configuration/Options.cs | 19 +++++++++---------- libs/host/Configuration/OptionsValidators.cs | 18 +++++++++++------- libs/server/Servers/ServerOptions.cs | 5 +++++ 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/libs/host/Configuration/Options.cs b/libs/host/Configuration/Options.cs index f002f31f30a..afd46cef520 100644 --- a/libs/host/Configuration/Options.cs +++ b/libs/host/Configuration/Options.cs @@ -40,8 +40,8 @@ internal sealed class Options public int Port { get; set; } [IpAddressValidation(false)] - [Option("bind", Required = false, HelpText = "IP address to bind server to (default: any)")] - public string Address { get; set; } + [Option("bind", Separator = ',', Required = false, HelpText = "IP address to bind server to (default: any)")] + public IEnumerable Address { get; set; } [MemorySizeValidation] [Option('m', "memory", Required = false, HelpText = "Total log memory used in bytes (rounds down to power of 2)")] @@ -616,9 +616,6 @@ public bool IsValid(out List invalidOptions, ILogger logger = null) this.runtimeLogger = logger; foreach (var prop in typeof(Options).GetProperties()) { - if (prop.Name.Equals("runtimeLogger")) - continue; - // Ignore if property is not decorated with the OptionsAttribute or the ValidationAttribute var validationAttr = prop.GetCustomAttributes(typeof(ValidationAttribute)).FirstOrDefault(); if (!Attribute.IsDefined(prop, typeof(OptionAttribute)) || validationAttr == null) @@ -658,15 +655,16 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null) var checkpointDir = CheckpointDir; if (!useAzureStorage) checkpointDir = new DirectoryInfo(string.IsNullOrEmpty(checkpointDir) ? (string.IsNullOrEmpty(logDir) ? "." : logDir) : checkpointDir).FullName; - EndPoint endpoint; + EndPoint[] endpoints; if (!string.IsNullOrEmpty(UnixSocketPath)) { - endpoint = new UnixDomainSocketEndPoint(UnixSocketPath); + endpoints = new EndPoint[1]; + endpoints[0] = new UnixDomainSocketEndPoint(UnixSocketPath); } else { - endpoint = Format.TryCreateEndpoint(Address, Port, useForBind: false).Result; - if (endpoint == null) + endpoints = Address.Select(Address => Format.TryCreateEndpoint(Address, Port, useForBind: false).Result).ToArray(); + if (endpoints.Length == 0) throw new GarnetException($"Invalid endpoint format {Address} {Port}."); } @@ -722,7 +720,8 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null) } return new GarnetServerOptions(logger) { - EndPoint = endpoint, + EndPoint = endpoints[0], + EndPoints = endpoints, MemorySize = MemorySize, PageSize = PageSize, SegmentSize = SegmentSize, diff --git a/libs/host/Configuration/OptionsValidators.cs b/libs/host/Configuration/OptionsValidators.cs index 1ff2d35656e..ad28b7f5a8d 100644 --- a/libs/host/Configuration/OptionsValidators.cs +++ b/libs/host/Configuration/OptionsValidators.cs @@ -357,17 +357,21 @@ internal IpAddressValidationAttribute(bool isRequired = true) : base(isRequired) /// protected override ValidationResult IsValid(object value, ValidationContext validationContext) { - if (TryInitialValidation(value, validationContext, out var initValidationResult, out var ipAddress)) + if (TryInitialValidation(value, validationContext, out var initValidationResult, out var ipAddresses)) return initValidationResult; var logger = ((Options)validationContext.ObjectInstance).runtimeLogger; - if (ipAddress.Equals(Localhost, StringComparison.CurrentCultureIgnoreCase) || - Format.TryCreateEndpoint(ipAddress, 0, useForBind: false, logger: logger).Result != null) - return ValidationResult.Success; + foreach (var ipAddress in ipAddresses) + { + if (ipAddress.Equals(Localhost, StringComparison.CurrentCultureIgnoreCase) || Format.TryCreateEndpoint(ipAddress, 0, useForBind: false, logger: logger).Result != null) + continue; - var baseError = validationContext.MemberName != null ? base.FormatErrorMessage(validationContext.MemberName) : string.Empty; - var errorMessage = $"{baseError} Expected string in IPv4 / IPv6 format (e.g. 127.0.0.1 / 0:0:0:0:0:0:0:1) or 'localhost' or valid hostname. Actual value: {ipAddress}"; - return new ValidationResult(errorMessage, [validationContext.MemberName]); + var baseError = validationContext.MemberName != null ? base.FormatErrorMessage(validationContext.MemberName) : string.Empty; + var errorMessage = $"{baseError} Expected string in IPv4 / IPv6 format (e.g. 127.0.0.1 / 0:0:0:0:0:0:0:1) or 'localhost' or valid hostname. Actual value: {ipAddress}"; + return new ValidationResult(errorMessage, [validationContext.MemberName]); + } + + return ValidationResult.Success; } } diff --git a/libs/server/Servers/ServerOptions.cs b/libs/server/Servers/ServerOptions.cs index 297b257657e..7d9178e7fbb 100644 --- a/libs/server/Servers/ServerOptions.cs +++ b/libs/server/Servers/ServerOptions.cs @@ -19,6 +19,11 @@ public class ServerOptions /// public EndPoint EndPoint { get; set; } = new IPEndPoint(IPAddress.Loopback, 6379); + /// + /// Endpoints to bind server to. + /// + public EndPoint[] EndPoints { get; set; } = [new IPEndPoint(IPAddress.Loopback, 6379)]; + /// /// Total log memory used in bytes (rounds down to power of 2). /// From a4d9738927c98451d221ed0693b1d5ef22dddcdb Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Wed, 5 Mar 2025 11:24:28 -0800 Subject: [PATCH 02/26] return list of endpoints from hostname value --- libs/cluster/Server/Gossip.cs | 6 +++--- libs/common/Format.cs | 21 ++++++++------------- libs/host/Configuration/Options.cs | 2 +- 3 files changed, 12 insertions(+), 17 deletions(-) diff --git a/libs/cluster/Server/Gossip.cs b/libs/cluster/Server/Gossip.cs index ee29c41b93c..35d34438a9d 100644 --- a/libs/cluster/Server/Gossip.cs +++ b/libs/cluster/Server/Gossip.cs @@ -161,12 +161,12 @@ public async Task TryMeetAsync(string address, int port, bool acquireLock = true if (gsn == null) { - var endpoint = await Format.TryCreateEndpoint(address, port, useForBind: true, logger: logger); - if (endpoint == null) + var endpoints = await Format.TryCreateEndpoint(address, port, useForBind: true, logger: logger); + if (endpoints == null) { logger?.LogError("Could not parse endpoint {address} {port}", address, port); } - gsn = new GarnetServerNode(clusterProvider, endpoint, tlsOptions?.TlsClientOptions, logger: logger); + gsn = new GarnetServerNode(clusterProvider, endpoints[0], tlsOptions?.TlsClientOptions, logger: logger); created = true; } diff --git a/libs/common/Format.cs b/libs/common/Format.cs index defb3f5d27c..bdac24a6c62 100644 --- a/libs/common/Format.cs +++ b/libs/common/Format.cs @@ -3,6 +3,7 @@ using System; using System.Diagnostics.CodeAnalysis; +using System.Linq; using System.Net; using System.Net.Sockets; using System.Threading.Tasks; @@ -39,14 +40,13 @@ public static class Format /// Binding does not poll connection because is supposed to be called from the server side /// /// - public static async Task TryCreateEndpoint(string addressOrHostname, int port, bool useForBind = false, ILogger logger = null) + public static async Task TryCreateEndpoint(string addressOrHostname, int port, bool useForBind = false, ILogger logger = null) { - IPEndPoint endpoint = null; if (string.IsNullOrEmpty(addressOrHostname) || string.IsNullOrWhiteSpace(addressOrHostname)) - return new IPEndPoint(IPAddress.Any, port); + return [new IPEndPoint(IPAddress.Any, port)]; if (IPAddress.TryParse(addressOrHostname, out var ipAddress)) - return new IPEndPoint(ipAddress, port); + return [new IPEndPoint(ipAddress, port)]; // Sanity check, there should be at least one ip address available try @@ -62,9 +62,9 @@ public static async Task TryCreateEndpoint(string addressOrHostname, i { foreach (var entry in ipAddresses) { - endpoint = new IPEndPoint(entry, port); + var endpoint = new IPEndPoint(entry, port); var IsListening = await IsReachable(endpoint); - if (IsListening) break; + if (IsListening) return [endpoint]; } } else @@ -78,12 +78,7 @@ public static async Task TryCreateEndpoint(string addressOrHostname, i return null; } - if (ipAddresses.Length > 1) { - logger?.LogError("Error hostname resolved to multiple endpoints. Garnet does not support multiple endpoints!"); - return null; - } - - return new IPEndPoint(ipAddresses[0], port); + return ipAddresses.Select(ip => new IPEndPoint(ip, port)).ToArray(); } logger?.LogError("No reachable IP address found for hostname:{hostname}", addressOrHostname); } @@ -92,7 +87,7 @@ public static async Task TryCreateEndpoint(string addressOrHostname, i logger?.LogError(ex, "Error while trying to resolve hostname:{hostname}", addressOrHostname); } - return endpoint; + return null; async Task IsReachable(IPEndPoint endpoint) { diff --git a/libs/host/Configuration/Options.cs b/libs/host/Configuration/Options.cs index afd46cef520..c57fc9601af 100644 --- a/libs/host/Configuration/Options.cs +++ b/libs/host/Configuration/Options.cs @@ -663,7 +663,7 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null) } else { - endpoints = Address.Select(Address => Format.TryCreateEndpoint(Address, Port, useForBind: false).Result).ToArray(); + endpoints = Address.SelectMany(Address => Format.TryCreateEndpoint(Address, Port, useForBind: false).Result).ToArray(); if (endpoints.Length == 0) throw new GarnetException($"Invalid endpoint format {Address} {Port}."); } From 8b10e61e88bab3db742a353c38ccb12a77052386 Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Wed, 5 Mar 2025 16:03:18 -0800 Subject: [PATCH 03/26] change endpoint to endpoints in Options.cs --- .../BDN.benchmark/Cluster/ClusterContext.cs | 2 +- libs/cluster/Server/ClusterManager.cs | 2 +- .../Configuration/GarnetCustomTransformers.cs | 22 ------------- libs/host/Configuration/Options.cs | 3 +- .../Redis/RedisConfigSerializer.cs | 33 +++++++------------ libs/host/Configuration/Redis/RedisOptions.cs | 6 ++-- libs/host/GarnetServer.cs | 8 ++--- libs/server/Servers/ServerOptions.cs | 5 --- test/Garnet.test/GarnetServerConfigTests.cs | 2 +- test/Garnet.test/TestUtils.cs | 6 ++-- 10 files changed, 26 insertions(+), 63 deletions(-) diff --git a/benchmark/BDN.benchmark/Cluster/ClusterContext.cs b/benchmark/BDN.benchmark/Cluster/ClusterContext.cs index fcfc9a265c9..1b2bd245be8 100644 --- a/benchmark/BDN.benchmark/Cluster/ClusterContext.cs +++ b/benchmark/BDN.benchmark/Cluster/ClusterContext.cs @@ -35,7 +35,7 @@ public void SetupSingleInstance(bool disableSlotVerification = false) { QuietMode = true, EnableCluster = !disableSlotVerification, - EndPoint = new IPEndPoint(IPAddress.Loopback, port), + EndPoints = [new IPEndPoint(IPAddress.Loopback, port)], CleanClusterConfig = true, }; if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) diff --git a/libs/cluster/Server/ClusterManager.cs b/libs/cluster/Server/ClusterManager.cs index 9d95c4d6657..bc26615bd0f 100644 --- a/libs/cluster/Server/ClusterManager.cs +++ b/libs/cluster/Server/ClusterManager.cs @@ -50,7 +50,7 @@ public unsafe ClusterManager(ClusterProvider clusterProvider, ILogger logger = n clusterConfigDevice = deviceFactory.Get(new FileDescriptor(directoryName: "", fileName: "nodes.conf")); pool = new(1, (int)clusterConfigDevice.SectorSize); - if (opts.EndPoint is not IPEndPoint endpoint) + if (opts.EndPoints[0] is not IPEndPoint endpoint) throw new NotImplementedException("Cluster mode for unix domain sockets has not been implemented."); var address = clusterProvider.storeWrapper.GetIp(); diff --git a/libs/host/Configuration/GarnetCustomTransformers.cs b/libs/host/Configuration/GarnetCustomTransformers.cs index da184488086..260d6a6ac96 100644 --- a/libs/host/Configuration/GarnetCustomTransformers.cs +++ b/libs/host/Configuration/GarnetCustomTransformers.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; namespace Garnet { @@ -66,27 +65,6 @@ public bool TransformBack(string input, out string output, out string errorMessa } } - /// - /// Transforms an array of type T to an object of type T by taking only the first element in the array - /// - /// The type of the array - internal class ArrayToFirstItemTransformer : IGarnetCustomTransformer - { - public bool Transform(T[] input, out T output, out string errorMessage) - { - errorMessage = null; - output = input == null || input.Length == 0 ? default : input.First(); - return true; - } - - public bool TransformBack(T input, out T[] output, out string errorMessage) - { - errorMessage = null; - output = [input]; - return true; - } - } - /// /// Transforms an object of type T to a nullable boolean by setting boolean to True if object is non-default, and False otherwise /// diff --git a/libs/host/Configuration/Options.cs b/libs/host/Configuration/Options.cs index c57fc9601af..8c501aceba0 100644 --- a/libs/host/Configuration/Options.cs +++ b/libs/host/Configuration/Options.cs @@ -40,7 +40,7 @@ internal sealed class Options public int Port { get; set; } [IpAddressValidation(false)] - [Option("bind", Separator = ',', Required = false, HelpText = "IP address to bind server to (default: any)")] + [Option("bind", Separator = ' ', Required = false, HelpText = "IP address to bind server to (default: any)")] public IEnumerable Address { get; set; } [MemorySizeValidation] @@ -720,7 +720,6 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null) } return new GarnetServerOptions(logger) { - EndPoint = endpoints[0], EndPoints = endpoints, MemorySize = MemorySize, PageSize = PageSize, diff --git a/libs/host/Configuration/Redis/RedisConfigSerializer.cs b/libs/host/Configuration/Redis/RedisConfigSerializer.cs index 832299bbf49..5ab9fdb1d82 100644 --- a/libs/host/Configuration/Redis/RedisConfigSerializer.cs +++ b/libs/host/Configuration/Redis/RedisConfigSerializer.cs @@ -95,30 +95,21 @@ public static RedisOptions Deserialize(StreamReader reader, ILogger logger) if (!TryChangeType(value, typeof(string), optType, out var newVal)) { // If unsuccessful and if underlying option type is an array, try to deserialize array by elements - if (optType.IsArray) - { - // Split the values in the serialized array - var values = value.Split(' '); - - // Instantiate a new array - var elemType = optType.GetElementType(); - newVal = Array.CreateInstance(elemType, values.Length); - - // Try deserializing and setting array elements - for (var i = 0; i < values.Length; i++) - { - if (!TryChangeType(values[i], typeof(string), elemType, out var elem)) - throw new RedisSerializationException( - $"Unable to deserialize {nameof(RedisOptions)} object. Unable to convert object of type {typeof(string)} to object of type {elemType}. (Line: {lineCount}; Key: {key}; Property: {prop.Name})."); + // Split the values in the serialized array + var values = value.Split(' '); + // Instantiate a new array + var elemType = optType.GenericTypeArguments.First(); + newVal = Array.CreateInstance(elemType, values.Length); - ((Array)newVal).SetValue(elem, i); - } - } - else + // Try deserializing and setting array elements + for (var i = 0; i < values.Length; i++) { - throw new RedisSerializationException( - $"Unable to deserialize {nameof(RedisOptions)} object. Unable to convert object of type {typeof(string)} to object of type {optType}. (Line: {lineCount}; Key: {key}; Property: {prop.Name})."); + if (!TryChangeType(values[i], typeof(string), elemType, out var elem)) + throw new RedisSerializationException( + $"Unable to deserialize {nameof(RedisOptions)} object. Unable to convert object of type {typeof(string)} to object of type {elemType}. (Line: {lineCount}; Key: {key}; Property: {prop.Name})."); + + ((Array)newVal).SetValue(elem, i); } } diff --git a/libs/host/Configuration/Redis/RedisOptions.cs b/libs/host/Configuration/Redis/RedisOptions.cs index d3faa21e840..93c7f24db52 100644 --- a/libs/host/Configuration/Redis/RedisOptions.cs +++ b/libs/host/Configuration/Redis/RedisOptions.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. using System; +using System.Collections.Generic; namespace Garnet { @@ -18,14 +19,13 @@ namespace Garnet /// internal class RedisOptions { - private const string BindWarning = "Only first IP address specified in bind option is used. All other addresses are ignored."; private const string TlsPortWarning = "tls-port option is used only to determine if TLS is enabled. The port value is otherwise ignored."; private const string TlsCertFileWarning = @"When using tls-cert-file make sure to first convert your certificate format to .pfx. Specify your passphrase in the tls-key-file-pass option (or via the cert-password command line argument), if applicable. Specify your subject name via the cert-subject-name command line argument, if applicable."; - [RedisOption("bind", nameof(Options.Address), BindWarning, typeof(ArrayToFirstItemTransformer))] - public Option Bind { get; set; } + [RedisOption("bind", nameof(Options.Address))] + public Option> Bind { get; set; } [RedisOption("enable-debug-command", nameof(Options.EnableDebugCommand))] public Option EnableDebugCommand { get; set; } diff --git a/libs/host/GarnetServer.cs b/libs/host/GarnetServer.cs index 436ac16f11b..464035e20d4 100644 --- a/libs/host/GarnetServer.cs +++ b/libs/host/GarnetServer.cs @@ -175,7 +175,7 @@ private void InitializeServer() Console.WriteLine($""" {red} _________ /_||___||_\ {normal}Garnet {version} {(IntPtr.Size == 8 ? "64" : "32")} bit; {(opts.EnableCluster ? "cluster" : "standalone")} mode{red} - '. \ / .' {normal}Listening on: {opts.EndPoint}{red} + '. \ / .' {normal}Listening on: {opts.EndPoints[0]}{red} '.\ /.' {magenta}https://aka.ms/GetGarnet{red} '.' {normal} @@ -185,7 +185,7 @@ private void InitializeServer() var clusterFactory = opts.EnableCluster ? new ClusterFactory() : null; this.logger = this.loggerFactory?.CreateLogger("GarnetServer"); - logger?.LogInformation("Garnet {version} {bits} bit; {clusterMode} mode; Endpoint: {endpoint}", version, IntPtr.Size == 8 ? "64" : "32", opts.EnableCluster ? "cluster" : "standalone", opts.EndPoint); + logger?.LogInformation("Garnet {version} {bits} bit; {clusterMode} mode; Endpoint: {endpoint}", version, IntPtr.Size == 8 ? "64" : "32", opts.EnableCluster ? "cluster" : "standalone", opts.EndPoints[0]); logger?.LogInformation("Environment .NET {netVersion}; {osPlatform}; {processArch}", Environment.Version, Environment.OSVersion.Platform, RuntimeInformation.ProcessArchitecture); // Flush initialization logs from memory logger @@ -248,14 +248,14 @@ private void InitializeServer() logger.LogInformation("Total configured memory limit: {configMemoryLimit}", configMemoryLimit); } - if (opts.EndPoint is UnixDomainSocketEndPoint) + if (opts.EndPoints[0] is UnixDomainSocketEndPoint) { // Delete existing unix socket file, if it exists. File.Delete(opts.UnixSocketPath); } // Create Garnet TCP server if none was provided. - this.server ??= new GarnetServerTcp(opts.EndPoint, 0, opts.TlsOptions, opts.NetworkSendThrottleMax, opts.NetworkConnectionLimit, opts.UnixSocketPath, opts.UnixSocketPermission, logger); + this.server ??= new GarnetServerTcp(opts.EndPoints[0], 0, opts.TlsOptions, opts.NetworkSendThrottleMax, opts.NetworkConnectionLimit, opts.UnixSocketPath, opts.UnixSocketPermission, logger); storeWrapper = new StoreWrapper(version, RedisProtocolVersion, server, store, objectStore, objectStoreSizeTracker, customCommandManager, appendOnlyFile, opts, subscribeBroker, clusterFactory: clusterFactory, loggerFactory: loggerFactory); diff --git a/libs/server/Servers/ServerOptions.cs b/libs/server/Servers/ServerOptions.cs index 7d9178e7fbb..bcf61ee1d72 100644 --- a/libs/server/Servers/ServerOptions.cs +++ b/libs/server/Servers/ServerOptions.cs @@ -14,11 +14,6 @@ namespace Garnet.server /// public class ServerOptions { - /// - /// Endpoint to bind server to. - /// - public EndPoint EndPoint { get; set; } = new IPEndPoint(IPAddress.Loopback, 6379); - /// /// Endpoints to bind server to. /// diff --git a/test/Garnet.test/GarnetServerConfigTests.cs b/test/Garnet.test/GarnetServerConfigTests.cs index 6a49537fbef..682d236932e 100644 --- a/test/Garnet.test/GarnetServerConfigTests.cs +++ b/test/Garnet.test/GarnetServerConfigTests.cs @@ -164,7 +164,7 @@ public void ImportExportRedisConfigLocal() var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, silentMode: true); ClassicAssert.IsTrue(parseSuccessful); ClassicAssert.AreEqual(invalidOptions.Count, 0); - ClassicAssert.AreEqual("127.0.0.1", options.Address); + ClassicAssert.AreEqual("127.0.0.1", options.Address.First()); ClassicAssert.AreEqual(ConnectionProtectionOption.Local, options.EnableDebugCommand); ClassicAssert.AreEqual(6379, options.Port); ClassicAssert.AreEqual("20gb", options.MemorySize); diff --git a/test/Garnet.test/TestUtils.cs b/test/Garnet.test/TestUtils.cs index 8e66b69a771..e8ee59f5fb4 100644 --- a/test/Garnet.test/TestUtils.cs +++ b/test/Garnet.test/TestUtils.cs @@ -302,7 +302,7 @@ public static GarnetServer CreateGarnetServer( EnableStorageTier = logCheckpointDir != null, LogDir = logDir, CheckpointDir = checkpointDir, - EndPoint = endpoint ?? EndPoint, + EndPoints = [endpoint ?? EndPoint], DisablePubSub = disablePubSub, Recover = tryRecover, IndexSize = indexSize, @@ -500,7 +500,7 @@ public static GarnetServer[] CreateGarnetCluster( ClassicAssert.IsNotNull(opts); - if (opts.EndPoint is IPEndPoint ipEndpoint) + if (opts.EndPoints[0] is IPEndPoint ipEndpoint) { var iter = 0; while (!IsPortAvailable(ipEndpoint.Port)) @@ -610,7 +610,7 @@ public static GarnetServerOptions GetGarnetServerOptions( EnableStorageTier = useAzureStorage || (!disableStorageTier && logDir != null), LogDir = disableStorageTier ? null : logDir, CheckpointDir = checkpointDir, - EndPoint = endpoint, + EndPoints = [endpoint], DisablePubSub = disablePubSub, DisableObjects = disableObjects, EnableDebugCommand = ConnectionProtectionOption.Yes, From a9182103c91cb29b2079d050bcef064ec0171180 Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Fri, 7 Mar 2025 11:37:06 -0800 Subject: [PATCH 04/26] switch back to string and do explicit split during address validation --- libs/common/Format.cs | 64 ++++++++++++++++--- .../Configuration/GarnetCustomTransformers.cs | 22 +++++++ libs/host/Configuration/Options.cs | 7 +- libs/host/Configuration/OptionsValidators.cs | 10 ++- .../Redis/RedisConfigSerializer.cs | 33 ++++++---- libs/host/Configuration/Redis/RedisOptions.cs | 6 +- test/Garnet.test/GarnetServerConfigTests.cs | 2 +- 7 files changed, 108 insertions(+), 36 deletions(-) diff --git a/libs/common/Format.cs b/libs/common/Format.cs index bdac24a6c62..bb74969215a 100644 --- a/libs/common/Format.cs +++ b/libs/common/Format.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. using System; +using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Net; @@ -32,29 +33,72 @@ internal static bool IsNullOrWhiteSpace([NotNullWhen(false)] this string s) => #pragma warning disable format public static class Format { + /// + /// Parse address list string containing address separated by whitespace + /// + /// + /// + /// + /// + /// + /// + /// True if parse and address validation was successful, otherwise false + public static bool TryParseAddressList(string addressList, int port, out EndPoint[] endpoints, out string errorHostnameOrAddress, bool useForBind = false, ILogger logger = null) + { + endpoints = null; + errorHostnameOrAddress = null; + // Check if input null or empty + if (string.IsNullOrEmpty(addressList) || string.IsNullOrWhiteSpace(addressList)) + { + endpoints = [new IPEndPoint(IPAddress.Any, port)]; + return true; + } + + var addresses = addressList.Split(' '); + var endpointList = new List(); + // Validate addresses and create endpoints + foreach (var singleAddressOrHostname in addresses) + { + var e = TryCreateEndpoint(singleAddressOrHostname, port, useForBind, logger).Result; + if(e == null) + { + endpoints = null; + errorHostnameOrAddress = singleAddressOrHostname; + return false; + } + endpointList.AddRange(e); + } + endpoints = [.. endpointList]; + + return true; + } + /// /// Try to create an endpoint from address and port /// - /// This could be an address or a hostname that the method tries to resolve + /// This could be an address or a hostname that the method tries to resolve /// /// Binding does not poll connection because is supposed to be called from the server side /// /// - public static async Task TryCreateEndpoint(string addressOrHostname, int port, bool useForBind = false, ILogger logger = null) + public static async Task TryCreateEndpoint(string singleAddressOrHostname, int port, bool useForBind = false, ILogger logger = null) { - if (string.IsNullOrEmpty(addressOrHostname) || string.IsNullOrWhiteSpace(addressOrHostname)) + if (string.IsNullOrEmpty(singleAddressOrHostname) || string.IsNullOrWhiteSpace(singleAddressOrHostname)) return [new IPEndPoint(IPAddress.Any, port)]; - if (IPAddress.TryParse(addressOrHostname, out var ipAddress)) + if (singleAddressOrHostname.Equals("localhost", StringComparison.CurrentCultureIgnoreCase)) + return [new IPEndPoint(IPAddress.Loopback, port)]; + + if (IPAddress.TryParse(singleAddressOrHostname, out var ipAddress)) return [new IPEndPoint(ipAddress, port)]; // Sanity check, there should be at least one ip address available try { - var ipAddresses = Dns.GetHostAddresses(addressOrHostname); + var ipAddresses = Dns.GetHostAddresses(singleAddressOrHostname); if (ipAddresses.Length == 0) { - logger?.LogError("No IP address found for hostname:{hostname}", addressOrHostname); + logger?.LogError("No IP address found for hostname:{hostname}", singleAddressOrHostname); return null; } @@ -72,19 +116,19 @@ public static async Task TryCreateEndpoint(string addressOrHostname, var machineHostname = GetHostName(); // Hostname does match the one acquired from machine name - if (!addressOrHostname.Equals(machineHostname, StringComparison.OrdinalIgnoreCase)) + if (!singleAddressOrHostname.Equals(machineHostname, StringComparison.OrdinalIgnoreCase)) { - logger?.LogError("Provided hostname does not much acquired machine name {addressOrHostname} {machineHostname}!", addressOrHostname, machineHostname); + logger?.LogError("Provided hostname does not much acquired machine name {addressOrHostname} {machineHostname}!", singleAddressOrHostname, machineHostname); return null; } return ipAddresses.Select(ip => new IPEndPoint(ip, port)).ToArray(); } - logger?.LogError("No reachable IP address found for hostname:{hostname}", addressOrHostname); + logger?.LogError("No reachable IP address found for hostname:{hostname}", singleAddressOrHostname); } catch (Exception ex) { - logger?.LogError(ex, "Error while trying to resolve hostname:{hostname}", addressOrHostname); + logger?.LogError(ex, "Error while trying to resolve hostname:{hostname}", singleAddressOrHostname); } return null; diff --git a/libs/host/Configuration/GarnetCustomTransformers.cs b/libs/host/Configuration/GarnetCustomTransformers.cs index 260d6a6ac96..da184488086 100644 --- a/libs/host/Configuration/GarnetCustomTransformers.cs +++ b/libs/host/Configuration/GarnetCustomTransformers.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; namespace Garnet { @@ -65,6 +66,27 @@ public bool TransformBack(string input, out string output, out string errorMessa } } + /// + /// Transforms an array of type T to an object of type T by taking only the first element in the array + /// + /// The type of the array + internal class ArrayToFirstItemTransformer : IGarnetCustomTransformer + { + public bool Transform(T[] input, out T output, out string errorMessage) + { + errorMessage = null; + output = input == null || input.Length == 0 ? default : input.First(); + return true; + } + + public bool TransformBack(T input, out T[] output, out string errorMessage) + { + errorMessage = null; + output = [input]; + return true; + } + } + /// /// Transforms an object of type T to a nullable boolean by setting boolean to True if object is non-default, and False otherwise /// diff --git a/libs/host/Configuration/Options.cs b/libs/host/Configuration/Options.cs index 8c501aceba0..c0290d68360 100644 --- a/libs/host/Configuration/Options.cs +++ b/libs/host/Configuration/Options.cs @@ -40,8 +40,8 @@ internal sealed class Options public int Port { get; set; } [IpAddressValidation(false)] - [Option("bind", Separator = ' ', Required = false, HelpText = "IP address to bind server to (default: any)")] - public IEnumerable Address { get; set; } + [Option("bind", Required = false, HelpText = "IP address to bind server to (default: any)")] + public string Address { get; set; } [MemorySizeValidation] [Option('m', "memory", Required = false, HelpText = "Total log memory used in bytes (rounds down to power of 2)")] @@ -663,8 +663,7 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null) } else { - endpoints = Address.SelectMany(Address => Format.TryCreateEndpoint(Address, Port, useForBind: false).Result).ToArray(); - if (endpoints.Length == 0) + if (!Format.TryParseAddressList(Address, Port, out endpoints, out _) || endpoints.Length == 0) throw new GarnetException($"Invalid endpoint format {Address} {Port}."); } diff --git a/libs/host/Configuration/OptionsValidators.cs b/libs/host/Configuration/OptionsValidators.cs index ad28b7f5a8d..392cf367e1a 100644 --- a/libs/host/Configuration/OptionsValidators.cs +++ b/libs/host/Configuration/OptionsValidators.cs @@ -357,17 +357,15 @@ internal IpAddressValidationAttribute(bool isRequired = true) : base(isRequired) /// protected override ValidationResult IsValid(object value, ValidationContext validationContext) { - if (TryInitialValidation(value, validationContext, out var initValidationResult, out var ipAddresses)) + if (TryInitialValidation(value, validationContext, out var initValidationResult, out var ipAddresses)) return initValidationResult; var logger = ((Options)validationContext.ObjectInstance).runtimeLogger; - foreach (var ipAddress in ipAddresses) - { - if (ipAddress.Equals(Localhost, StringComparison.CurrentCultureIgnoreCase) || Format.TryCreateEndpoint(ipAddress, 0, useForBind: false, logger: logger).Result != null) - continue; + if (!Format.TryParseAddressList(ipAddresses, 0, out var endpoints, out var errorHostnameOrAddress, useForBind: false, logger: logger)) + { var baseError = validationContext.MemberName != null ? base.FormatErrorMessage(validationContext.MemberName) : string.Empty; - var errorMessage = $"{baseError} Expected string in IPv4 / IPv6 format (e.g. 127.0.0.1 / 0:0:0:0:0:0:0:1) or 'localhost' or valid hostname. Actual value: {ipAddress}"; + var errorMessage = $"{baseError} Expected string in IPv4 / IPv6 format (e.g. 127.0.0.1 / 0:0:0:0:0:0:0:1) or 'localhost' or valid hostname. Actual value: {errorHostnameOrAddress}"; return new ValidationResult(errorMessage, [validationContext.MemberName]); } diff --git a/libs/host/Configuration/Redis/RedisConfigSerializer.cs b/libs/host/Configuration/Redis/RedisConfigSerializer.cs index 5ab9fdb1d82..832299bbf49 100644 --- a/libs/host/Configuration/Redis/RedisConfigSerializer.cs +++ b/libs/host/Configuration/Redis/RedisConfigSerializer.cs @@ -95,21 +95,30 @@ public static RedisOptions Deserialize(StreamReader reader, ILogger logger) if (!TryChangeType(value, typeof(string), optType, out var newVal)) { // If unsuccessful and if underlying option type is an array, try to deserialize array by elements - // Split the values in the serialized array - var values = value.Split(' '); + if (optType.IsArray) + { + // Split the values in the serialized array + var values = value.Split(' '); - // Instantiate a new array - var elemType = optType.GenericTypeArguments.First(); - newVal = Array.CreateInstance(elemType, values.Length); + // Instantiate a new array + var elemType = optType.GetElementType(); + newVal = Array.CreateInstance(elemType, values.Length); + + // Try deserializing and setting array elements + for (var i = 0; i < values.Length; i++) + { + if (!TryChangeType(values[i], typeof(string), elemType, out var elem)) + throw new RedisSerializationException( + $"Unable to deserialize {nameof(RedisOptions)} object. Unable to convert object of type {typeof(string)} to object of type {elemType}. (Line: {lineCount}; Key: {key}; Property: {prop.Name})."); - // Try deserializing and setting array elements - for (var i = 0; i < values.Length; i++) - { - if (!TryChangeType(values[i], typeof(string), elemType, out var elem)) - throw new RedisSerializationException( - $"Unable to deserialize {nameof(RedisOptions)} object. Unable to convert object of type {typeof(string)} to object of type {elemType}. (Line: {lineCount}; Key: {key}; Property: {prop.Name})."); - ((Array)newVal).SetValue(elem, i); + ((Array)newVal).SetValue(elem, i); + } + } + else + { + throw new RedisSerializationException( + $"Unable to deserialize {nameof(RedisOptions)} object. Unable to convert object of type {typeof(string)} to object of type {optType}. (Line: {lineCount}; Key: {key}; Property: {prop.Name})."); } } diff --git a/libs/host/Configuration/Redis/RedisOptions.cs b/libs/host/Configuration/Redis/RedisOptions.cs index 93c7f24db52..d3faa21e840 100644 --- a/libs/host/Configuration/Redis/RedisOptions.cs +++ b/libs/host/Configuration/Redis/RedisOptions.cs @@ -2,7 +2,6 @@ // Licensed under the MIT license. using System; -using System.Collections.Generic; namespace Garnet { @@ -19,13 +18,14 @@ namespace Garnet /// internal class RedisOptions { + private const string BindWarning = "Only first IP address specified in bind option is used. All other addresses are ignored."; private const string TlsPortWarning = "tls-port option is used only to determine if TLS is enabled. The port value is otherwise ignored."; private const string TlsCertFileWarning = @"When using tls-cert-file make sure to first convert your certificate format to .pfx. Specify your passphrase in the tls-key-file-pass option (or via the cert-password command line argument), if applicable. Specify your subject name via the cert-subject-name command line argument, if applicable."; - [RedisOption("bind", nameof(Options.Address))] - public Option> Bind { get; set; } + [RedisOption("bind", nameof(Options.Address), BindWarning, typeof(ArrayToFirstItemTransformer))] + public Option Bind { get; set; } [RedisOption("enable-debug-command", nameof(Options.EnableDebugCommand))] public Option EnableDebugCommand { get; set; } diff --git a/test/Garnet.test/GarnetServerConfigTests.cs b/test/Garnet.test/GarnetServerConfigTests.cs index 682d236932e..6a49537fbef 100644 --- a/test/Garnet.test/GarnetServerConfigTests.cs +++ b/test/Garnet.test/GarnetServerConfigTests.cs @@ -164,7 +164,7 @@ public void ImportExportRedisConfigLocal() var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, silentMode: true); ClassicAssert.IsTrue(parseSuccessful); ClassicAssert.AreEqual(invalidOptions.Count, 0); - ClassicAssert.AreEqual("127.0.0.1", options.Address.First()); + ClassicAssert.AreEqual("127.0.0.1", options.Address); ClassicAssert.AreEqual(ConnectionProtectionOption.Local, options.EnableDebugCommand); ClassicAssert.AreEqual(6379, options.Port); ClassicAssert.AreEqual("20gb", options.MemorySize); From 82537676770769586f11df84a3ba818b8bd23bab Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Mon, 10 Mar 2025 10:54:35 -0700 Subject: [PATCH 05/26] create multiple server tcp instances to support multiple endpoints --- .../Embedded/EmbeddedRespServer.cs | 2 +- libs/cluster/Server/ClusterProvider.cs | 28 +++---- libs/host/GarnetServer.cs | 26 ++++--- libs/server/Metrics/GarnetServerMonitor.cs | 78 +++++++++++-------- libs/server/Metrics/Info/GarnetInfoMetrics.cs | 5 +- libs/server/Resp/PurgeBPCommand.cs | 3 +- libs/server/StoreWrapper.cs | 12 +-- 7 files changed, 91 insertions(+), 63 deletions(-) diff --git a/benchmark/BDN.benchmark/Embedded/EmbeddedRespServer.cs b/benchmark/BDN.benchmark/Embedded/EmbeddedRespServer.cs index dbf4bdb70bb..3a0f7b2e0f7 100644 --- a/benchmark/BDN.benchmark/Embedded/EmbeddedRespServer.cs +++ b/benchmark/BDN.benchmark/Embedded/EmbeddedRespServer.cs @@ -21,7 +21,7 @@ internal sealed class EmbeddedRespServer : GarnetServer /// Server options to configure the base GarnetServer instance /// Logger factory to configure the base GarnetServer instance /// Server network - public EmbeddedRespServer(GarnetServerOptions opts, ILoggerFactory loggerFactory = null, GarnetServerEmbedded server = null) : base(opts, loggerFactory, server) + public EmbeddedRespServer(GarnetServerOptions opts, ILoggerFactory loggerFactory = null, GarnetServerEmbedded server = null) : base(opts, loggerFactory, [server]) { this.garnetServerEmbedded = server; this.subscribeBroker = opts.DisablePubSub ? null : diff --git a/libs/cluster/Server/ClusterProvider.cs b/libs/cluster/Server/ClusterProvider.cs index 790edcfd20a..7297aa33794 100644 --- a/libs/cluster/Server/ClusterProvider.cs +++ b/libs/cluster/Server/ClusterProvider.cs @@ -439,23 +439,25 @@ internal ReplicationLogCheckpointManager GetReplicationLogCheckpointManager(Stor /// internal bool BumpAndWaitForEpochTransition() { - var server = storeWrapper.TcpServer; BumpCurrentEpoch(); - while (true) + foreach (var server in storeWrapper.TcpServer) { - retry: - Thread.Yield(); - // Acquire latest bumped epoch - var currentEpoch = GarnetCurrentEpoch; - var sessions = server.ActiveClusterSessions(); - foreach (var s in sessions) + while (true) { - var entryEpoch = s.LocalCurrentEpoch; - // Retry if at least one session has not yet caught up to the current epoch. - if (entryEpoch != 0 && entryEpoch < currentEpoch) - goto retry; + retry: + Thread.Yield(); + // Acquire latest bumped epoch + var currentEpoch = GarnetCurrentEpoch; + var sessions = ((GarnetServerTcp)server).ActiveClusterSessions(); + foreach (var s in sessions) + { + var entryEpoch = s.LocalCurrentEpoch; + // Retry if at least one session has not yet caught up to the current epoch. + if (entryEpoch != 0 && entryEpoch < currentEpoch) + goto retry; + } + break; } - break; } return true; } diff --git a/libs/host/GarnetServer.cs b/libs/host/GarnetServer.cs index 464035e20d4..4db0e7e1ef2 100644 --- a/libs/host/GarnetServer.cs +++ b/libs/host/GarnetServer.cs @@ -46,7 +46,7 @@ static string GetVersion() internal GarnetProvider Provider; private readonly GarnetServerOptions opts; - private IGarnetServer server; + private IGarnetServer[] servers; private TsavoriteKV store; private TsavoriteKV objectStore; private IDevice aofDevice; @@ -151,11 +151,11 @@ public GarnetServer(string[] commandLineArgs, ILoggerFactory loggerFactory = nul /// /// Server options /// Logger factory - /// The IGarnetServer to use. If none is provided, will use a GarnetServerTcp. + /// The IGarnetServer to use. If none is provided, will use a GarnetServerTcp. /// Whether to clean up data folders on dispose - public GarnetServer(GarnetServerOptions opts, ILoggerFactory loggerFactory = null, IGarnetServer server = null, bool cleanupDir = false) + public GarnetServer(GarnetServerOptions opts, ILoggerFactory loggerFactory = null, IGarnetServer[] servers = null, bool cleanupDir = false) { - this.server = server; + this.servers = servers; this.opts = opts; this.loggerFactory = loggerFactory; this.cleanupDir = cleanupDir; @@ -255,9 +255,14 @@ private void InitializeServer() } // Create Garnet TCP server if none was provided. - this.server ??= new GarnetServerTcp(opts.EndPoints[0], 0, opts.TlsOptions, opts.NetworkSendThrottleMax, opts.NetworkConnectionLimit, opts.UnixSocketPath, opts.UnixSocketPermission, logger); + if (servers == null) + { + servers = new IGarnetServer[opts.EndPoints.Length]; + for (var i = 0; i < servers.Length; i++) + servers[i] = new GarnetServerTcp(opts.EndPoints[i], 0, opts.TlsOptions, opts.NetworkSendThrottleMax, opts.NetworkConnectionLimit, opts.UnixSocketPath, opts.UnixSocketPermission, logger); + } - storeWrapper = new StoreWrapper(version, RedisProtocolVersion, server, store, objectStore, objectStoreSizeTracker, + storeWrapper = new StoreWrapper(version, RedisProtocolVersion, servers, store, objectStore, objectStoreSizeTracker, customCommandManager, appendOnlyFile, opts, subscribeBroker, clusterFactory: clusterFactory, loggerFactory: loggerFactory); // Create session provider for Garnet @@ -268,7 +273,8 @@ private void InitializeServer() Register = new RegisterApi(Provider); Store = new StoreApi(storeWrapper); - server.Register(WireFormat.ASCII, Provider); + for (var i = 0; i < servers.Length; i++) + servers[i].Register(WireFormat.ASCII, Provider); LoadModules(customCommandManager); } @@ -381,7 +387,8 @@ private void CreateAOF() public void Start() { Provider.Recover(); - server.Start(); + for (var i = 0; i < servers.Length; i++) + servers[i].Start(); Provider.Start(); if (!opts.QuietMode) Console.WriteLine("* Ready to accept connections"); @@ -416,7 +423,8 @@ public void Dispose(bool deleteDir = true) private void InternalDispose() { Provider?.Dispose(); - server.Dispose(); + for (var i = 0; i < servers.Length; i++) + servers[i]?.Dispose(); subscribeBroker?.Dispose(); store.Dispose(); appendOnlyFile?.Dispose(); diff --git a/libs/server/Metrics/GarnetServerMonitor.cs b/libs/server/Metrics/GarnetServerMonitor.cs index 3ab5ab2c230..91c618dde8e 100644 --- a/libs/server/Metrics/GarnetServerMonitor.cs +++ b/libs/server/Metrics/GarnetServerMonitor.cs @@ -28,7 +28,7 @@ public readonly Dictionary readonly StoreWrapper storeWrapper; readonly GarnetServerOptions opts; - readonly IGarnetServer server; + readonly IGarnetServer[] servers; readonly TimeSpan monitorSamplingFrequency; public long monitor_iterations; @@ -48,11 +48,11 @@ public readonly Dictionary SingleWriterMultiReaderLock rwLock = new(); - public GarnetServerMonitor(StoreWrapper storeWrapper, GarnetServerOptions opts, IGarnetServer server, ILogger logger = null) + public GarnetServerMonitor(StoreWrapper storeWrapper, GarnetServerOptions opts, IGarnetServer[] servers, ILogger logger = null) { this.storeWrapper = storeWrapper; this.opts = opts; - this.server = server; + this.servers = servers; this.logger = logger; monitorSamplingFrequency = TimeSpan.FromSeconds(opts.MetricsSamplingFrequency); monitor_iterations = 0; @@ -94,14 +94,17 @@ public void AddMetricsHistorySessionDispose(GarnetSessionMetrics currSessionMetr public string GetAllLocksets() { - string result = ""; - var sessions = ((GarnetServerBase)server).ActiveConsumers(); - foreach (var s in sessions) + var result = ""; + foreach (var server in servers) { - var session = (RespServerSession)s; - var lockset = session.txnManager.GetLockset(); - if (lockset != "") - result += session.StoreSessionID + ": " + lockset + "\n"; + var sessions = ((GarnetServerBase)server).ActiveConsumers(); + foreach (var s in sessions) + { + var session = (RespServerSession)s; + var lockset = session.txnManager.GetLockset(); + if (lockset != "") + result += session.StoreSessionID + ": " + lockset + "\n"; + } } return result; } @@ -180,16 +183,18 @@ private void ResetStats() globalMetrics.globalSessionMetrics.Reset(); globalMetrics.historySessionMetrics.Reset(); - var garnetServer = ((GarnetServerBase)server); - var sessions = garnetServer.ActiveConsumers(); - foreach (var s in sessions) + foreach (var garnetServer in servers.Cast()) { - var session = (RespServerSession)s; - session.GetSessionMetrics.Reset(); - } + var sessions = garnetServer.ActiveConsumers(); + foreach (var s in sessions) + { + var session = (RespServerSession)s; + session.GetSessionMetrics.Reset(); + } - garnetServer.ResetConnectionsReceived(); - garnetServer.ResetConnectionsDiposed(); + garnetServer.ResetConnectionsReceived(); + garnetServer.ResetConnectionsDiposed(); + } storeWrapper.clusterProvider?.ResetGossipStats(); @@ -210,9 +215,12 @@ private void ResetLatencyMetrics() { logger?.LogInformation("Resetting server-side stats {eventType}", eventType); - var sessions = ((GarnetServerBase)server).ActiveConsumers(); - foreach (var entry in sessions) - ((RespServerSession)entry).ResetLatencyMetrics(eventType); + foreach (var server in servers) + { + var sessions = ((GarnetServerBase)server).ActiveConsumers(); + foreach (var entry in sessions) + ((RespServerSession)entry).ResetLatencyMetrics(eventType); + } rwLock.WriteLock(); try @@ -234,9 +242,12 @@ private void ResetLatencySessionMetrics() { if (opts.LatencyMonitor) { - var sessions = ((GarnetServerBase)server).ActiveConsumers(); - foreach (var entry in sessions) - ((RespServerSession)entry).ResetAllLatencyMetrics(); + foreach (var server in servers) + { + var sessions = ((GarnetServerBase)server).ActiveConsumers(); + foreach (var entry in sessions) + ((RespServerSession)entry).ResetAllLatencyMetrics(); + } } } @@ -255,14 +266,17 @@ private async void MainMonitorTask(CancellationToken token) monitor_iterations++; - var garnetServer = ((GarnetServerBase)server); - globalMetrics.total_connections_received = garnetServer.TotalConnectionsReceived; - globalMetrics.total_connections_disposed = garnetServer.TotalConnectionsDisposed; - globalMetrics.total_connections_active = garnetServer.get_conn_active(); - - UpdateInstantaneousMetrics(); - UpdateAllMetricsHistory(); - UpdateAllMetrics(server); + foreach (var server in servers) + { + var garnetServer = ((GarnetServerBase)server); + globalMetrics.total_connections_received = garnetServer.TotalConnectionsReceived; + globalMetrics.total_connections_disposed = garnetServer.TotalConnectionsDisposed; + globalMetrics.total_connections_active = garnetServer.get_conn_active(); + + UpdateInstantaneousMetrics(); + UpdateAllMetricsHistory(); + UpdateAllMetrics(server); + } //Reset & Cleanup ResetStats(); diff --git a/libs/server/Metrics/Info/GarnetInfoMetrics.cs b/libs/server/Metrics/Info/GarnetInfoMetrics.cs index 02e14589f5a..9baa6f0fd36 100644 --- a/libs/server/Metrics/Info/GarnetInfoMetrics.cs +++ b/libs/server/Metrics/Info/GarnetInfoMetrics.cs @@ -288,7 +288,10 @@ private void PopulateKeyspaceInfo(StoreWrapper storeWrapper) private void PopulateClusterBufferPoolStats(StoreWrapper storeWrapper) { - bufferPoolStats = [new("server_socket", storeWrapper.TcpServer.GetBufferPoolStats())]; + var server = storeWrapper.TcpServer; + bufferPoolStats = new MetricsItem[server.Length]; + for (var i = 0; i < server.Length; i++) + bufferPoolStats[i] = new($"server_socket_{i}", ((GarnetServerTcp)server[i]).GetBufferPoolStats()); if (storeWrapper.clusterProvider != null) bufferPoolStats = [.. bufferPoolStats, .. storeWrapper.clusterProvider.GetBufferPoolStats()]; } diff --git a/libs/server/Resp/PurgeBPCommand.cs b/libs/server/Resp/PurgeBPCommand.cs index efc51e119f7..054036f8b22 100644 --- a/libs/server/Resp/PurgeBPCommand.cs +++ b/libs/server/Resp/PurgeBPCommand.cs @@ -69,7 +69,8 @@ private bool NetworkPurgeBP() success = ClusterPurgeBufferPool(managerType); break; case ManagerType.ServerListener: - storeWrapper.TcpServer.Purge(); + foreach (var server in storeWrapper.TcpServer) + ((GarnetServerTcp)server).Purge(); break; default: success = false; diff --git a/libs/server/StoreWrapper.cs b/libs/server/StoreWrapper.cs index e8430c9a8aa..4ff4c85aff0 100644 --- a/libs/server/StoreWrapper.cs +++ b/libs/server/StoreWrapper.cs @@ -31,7 +31,7 @@ public sealed class StoreWrapper : IDisposable { internal readonly string version; internal readonly string redisProtocolVersion; - readonly IGarnetServer server; + readonly IGarnetServer[] servers; internal readonly long startupTime; /// @@ -59,7 +59,7 @@ public sealed class StoreWrapper : IDisposable /// /// Get server /// - public GarnetServerTcp TcpServer => (GarnetServerTcp)server; + public IGarnetServer[] TcpServer => servers; /// /// Access control list governing all commands @@ -130,7 +130,7 @@ public sealed class StoreWrapper : IDisposable public StoreWrapper( string version, string redisProtocolVersion, - IGarnetServer server, + IGarnetServer[] servers, TsavoriteKV store, TsavoriteKV objectStore, CacheSizeTracker objectStoreSizeTracker, @@ -145,7 +145,7 @@ public StoreWrapper( { this.version = version; this.redisProtocolVersion = redisProtocolVersion; - this.server = server; + this.servers = servers; this.startupTime = DateTimeOffset.UtcNow.Ticks; this.store = store; this.objectStore = objectStore; @@ -154,7 +154,7 @@ public StoreWrapper( this.subscribeBroker = subscribeBroker; lastSaveTime = DateTimeOffset.FromUnixTimeSeconds(0); this.customCommandManager = customCommandManager; - this.monitor = serverOptions.MetricsSamplingFrequency > 0 ? new GarnetServerMonitor(this, serverOptions, server, loggerFactory?.CreateLogger("GarnetServerMonitor")) : null; + this.monitor = serverOptions.MetricsSamplingFrequency > 0 ? new GarnetServerMonitor(this, serverOptions, this.servers, loggerFactory?.CreateLogger("GarnetServerMonitor")) : null; this.objectStoreSizeTracker = objectStoreSizeTracker; this.loggerFactory = loggerFactory; this.logger = loggerFactory?.CreateLogger("StoreWrapper"); @@ -225,7 +225,7 @@ public StoreWrapper( /// public string GetIp() { - if (TcpServer.EndPoint is not IPEndPoint localEndpoint) + if (((GarnetServerTcp)TcpServer[0]).EndPoint is not IPEndPoint localEndpoint) throw new NotImplementedException("Cluster mode for unix domain sockets has not been implemented"); if (localEndpoint.Address.Equals(IPAddress.Any)) From 28190dc03015908fb108d53f6306815fea7ea7ed Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Mon, 10 Mar 2025 12:08:57 -0700 Subject: [PATCH 06/26] add test for multi endpoints --- test/Garnet.test/GarnetClientTests.cs | 2 +- test/Garnet.test/GarnetServerConfigTests.cs | 27 +++++++++++++++++++++ test/Garnet.test/TestUtils.cs | 12 +++++---- test/Garnet.test/UnixSocketTests.cs | 6 ++--- 4 files changed, 38 insertions(+), 9 deletions(-) diff --git a/test/Garnet.test/GarnetClientTests.cs b/test/Garnet.test/GarnetClientTests.cs index bf21b491d73..a00bd64fae3 100644 --- a/test/Garnet.test/GarnetClientTests.cs +++ b/test/Garnet.test/GarnetClientTests.cs @@ -599,7 +599,7 @@ public async Task UnixSocket_Ping([Values] bool useTls) var unixSocketPath = "./unix-socket-ping-test.sock"; var unixSocketEndpoint = new UnixDomainSocketEndPoint(unixSocketPath); - using var server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, unixSocketEndpoint, enableTLS: useTls, unixSocketPath: unixSocketPath); + using var server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, [unixSocketEndpoint], enableTLS: useTls, unixSocketPath: unixSocketPath); server.Start(); using var db = TestUtils.GetGarnetClient(unixSocketEndpoint, useTLS: useTls); diff --git a/test/Garnet.test/GarnetServerConfigTests.cs b/test/Garnet.test/GarnetServerConfigTests.cs index 6a49537fbef..4e690feae48 100644 --- a/test/Garnet.test/GarnetServerConfigTests.cs +++ b/test/Garnet.test/GarnetServerConfigTests.cs @@ -5,9 +5,11 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Net; using System.Reflection; using System.Text.Json; using System.Text.Json.Serialization; +using System.Threading.Tasks; using CommandLine; using Garnet.common; using Garnet.server; @@ -717,5 +719,30 @@ public void UnixSocketPermission_InvalidPermissionFails() var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out _, out _, out _, silentMode: true); ClassicAssert.IsFalse(parseSuccessful); } + + [Test] + public async Task MultiEndpointTest() + { + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + var hostname = TestUtils.GetHostName(); + var addresses = Dns.GetHostAddresses(hostname); + + var endpoints = addresses.Select(address => new IPEndPoint(address, TestUtils.TestPort)).ToArray(); + ClassicAssert.Greater(endpoints.Length, 1, $"{nameof(MultiEndpointTest)} not enough network interfaces!"); + var server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, endpoints: endpoints); + server.Start(); + + var clients = endpoints.Select(endpoint => TestUtils.GetGarnetClientSession(endPoint: endpoint)).ToArray(); + foreach (var client in clients) + { + client.Connect(); + var result = await client.ExecuteAsync("PING"); + ClassicAssert.AreEqual("PONG", result); + client.Dispose(); + } + + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir); + } } } \ No newline at end of file diff --git a/test/Garnet.test/TestUtils.cs b/test/Garnet.test/TestUtils.cs index e8ee59f5fb4..9db6294d6e7 100644 --- a/test/Garnet.test/TestUtils.cs +++ b/test/Garnet.test/TestUtils.cs @@ -68,10 +68,12 @@ public IEnumerable GetData(IParameterInfo parameter) internal static class TestUtils { + public static readonly int TestPort = 33278; + /// /// Test server end point /// - public static EndPoint EndPoint = new IPEndPoint(IPAddress.Loopback, 33278); + public static EndPoint EndPoint = new IPEndPoint(IPAddress.Loopback, TestPort); /// /// Whether to use a test progress logger @@ -213,7 +215,7 @@ internal static void IgnoreIfNotRunningAzureTests() /// public static GarnetServer CreateGarnetServer( string logCheckpointDir, - EndPoint endpoint = null, + EndPoint[] endpoints = null, bool disablePubSub = false, bool tryRecover = false, bool lowMemory = false, @@ -302,7 +304,7 @@ public static GarnetServer CreateGarnetServer( EnableStorageTier = logCheckpointDir != null, LogDir = logDir, CheckpointDir = checkpointDir, - EndPoints = [endpoint ?? EndPoint], + EndPoints = endpoints, DisablePubSub = disablePubSub, Recover = tryRecover, IndexSize = indexSize, @@ -776,7 +778,7 @@ public static GarnetClient GetGarnetClient(EndPoint endpoint = null, bool useTLS return new GarnetClient(endpoint ?? EndPoint, sslOptions, recordLatency: recordLatency); } - public static GarnetClientSession GetGarnetClientSession(bool useTLS = false, bool recordLatency = false) + public static GarnetClientSession GetGarnetClientSession(bool useTLS = false, bool recordLatency = false, EndPoint endPoint = null) { SslClientAuthenticationOptions sslOptions = null; if (useTLS) @@ -789,7 +791,7 @@ public static GarnetClientSession GetGarnetClientSession(bool useTLS = false, bo RemoteCertificateValidationCallback = ValidateServerCertificate, }; } - return new GarnetClientSession(EndPoint, new(), tlsOptions: sslOptions); + return new GarnetClientSession(endPoint ?? EndPoint, new(), tlsOptions: sslOptions); } public static LightClientRequest CreateRequest(LightClient.OnResponseDelegateUnsafe onReceive = null, bool useTLS = false, CountResponseType countResponseType = CountResponseType.Tokens) diff --git a/test/Garnet.test/UnixSocketTests.cs b/test/Garnet.test/UnixSocketTests.cs index 0066f859d74..b2277958fad 100644 --- a/test/Garnet.test/UnixSocketTests.cs +++ b/test/Garnet.test/UnixSocketTests.cs @@ -39,7 +39,7 @@ public async Task Permission_SetPermissionMatches([Values] bool useTls) UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | UnixFileMode.GroupRead | UnixFileMode.GroupWrite | UnixFileMode.GroupExecute; // 770 - using var server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, unixSocketEndpoint, enableTLS: useTls, unixSocketPath: unixSocketPath, unixSocketPermission: unixSocketPermission); + using var server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, [unixSocketEndpoint], enableTLS: useTls, unixSocketPath: unixSocketPath, unixSocketPermission: unixSocketPermission); server.Start(); using var client = await ConnectionMultiplexer.ConnectAsync(TestUtils.GetConfig([unixSocketEndpoint], useTLS: useTls)); @@ -55,7 +55,7 @@ public async Task Ping_DoesNotThrow([Values] bool useTls) var unixSocketPath = "./unix-socket-ping-test.sock"; var unixSocketEndpoint = new UnixDomainSocketEndPoint(unixSocketPath); - using var server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, unixSocketEndpoint, enableTLS: useTls, unixSocketPath: unixSocketPath); + using var server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, [unixSocketEndpoint], enableTLS: useTls, unixSocketPath: unixSocketPath); server.Start(); using var client = await ConnectionMultiplexer.ConnectAsync(TestUtils.GetConfig([unixSocketEndpoint], useTLS: useTls)); @@ -71,7 +71,7 @@ public async Task SetGet_Equals([Values] bool useTls, [Values(256, 256 * 2048)] var unixSocketPath = "./unix-socket-set-get-test.sock"; var unixSocketEndpoint = new UnixDomainSocketEndPoint(unixSocketPath); - using var server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, unixSocketEndpoint, enableTLS: useTls, unixSocketPath: unixSocketPath); + using var server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, [unixSocketEndpoint], enableTLS: useTls, unixSocketPath: unixSocketPath); server.Start(); using var client = await ConnectionMultiplexer.ConnectAsync(TestUtils.GetConfig([unixSocketEndpoint], useTLS: useTls)); From 07a7f2bd1f56510f897474ee28dd34439868ccd5 Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Mon, 10 Mar 2025 13:11:34 -0700 Subject: [PATCH 07/26] fix testUtils regression --- test/Garnet.test/TestUtils.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Garnet.test/TestUtils.cs b/test/Garnet.test/TestUtils.cs index 9db6294d6e7..fd58f55a10a 100644 --- a/test/Garnet.test/TestUtils.cs +++ b/test/Garnet.test/TestUtils.cs @@ -304,7 +304,7 @@ public static GarnetServer CreateGarnetServer( EnableStorageTier = logCheckpointDir != null, LogDir = logDir, CheckpointDir = checkpointDir, - EndPoints = endpoints, + EndPoints = endpoints ?? ([EndPoint]), DisablePubSub = disablePubSub, Recover = tryRecover, IndexSize = indexSize, From ace8cbe49f12be9c30d1771c8d165ca13eeda1b9 Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Mon, 10 Mar 2025 14:03:41 -0700 Subject: [PATCH 08/26] addloopback test --- test/Garnet.test/GarnetServerConfigTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Garnet.test/GarnetServerConfigTests.cs b/test/Garnet.test/GarnetServerConfigTests.cs index 4e690feae48..997b0e6ed88 100644 --- a/test/Garnet.test/GarnetServerConfigTests.cs +++ b/test/Garnet.test/GarnetServerConfigTests.cs @@ -726,9 +726,9 @@ public async Task MultiEndpointTest() TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); var hostname = TestUtils.GetHostName(); var addresses = Dns.GetHostAddresses(hostname); + addresses = [.. addresses, IPAddress.IPv6Loopback, IPAddress.Loopback]; var endpoints = addresses.Select(address => new IPEndPoint(address, TestUtils.TestPort)).ToArray(); - ClassicAssert.Greater(endpoints.Length, 1, $"{nameof(MultiEndpointTest)} not enough network interfaces!"); var server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, endpoints: endpoints); server.Start(); From e0517a97767a9f20eb2152b5e188e87b951dcfa7 Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Tue, 11 Mar 2025 15:55:36 -0700 Subject: [PATCH 09/26] fix typo --- libs/common/Format.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/common/Format.cs b/libs/common/Format.cs index bb74969215a..64146e8d9f6 100644 --- a/libs/common/Format.cs +++ b/libs/common/Format.cs @@ -115,7 +115,7 @@ public static async Task TryCreateEndpoint(string singleAddressOrHos { var machineHostname = GetHostName(); - // Hostname does match the one acquired from machine name + // User-provided hostname does not match the machine hostname if (!singleAddressOrHostname.Equals(machineHostname, StringComparison.OrdinalIgnoreCase)) { logger?.LogError("Provided hostname does not much acquired machine name {addressOrHostname} {machineHostname}!", singleAddressOrHostname, machineHostname); From 19bfb16a0f027a120add60f09ac890ceee4741c8 Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Tue, 11 Mar 2025 16:51:32 -0700 Subject: [PATCH 10/26] logger exception message instead of stack trace --- libs/common/Format.cs | 2 +- libs/host/Configuration/OptionsValidators.cs | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/libs/common/Format.cs b/libs/common/Format.cs index 64146e8d9f6..c55d8e28866 100644 --- a/libs/common/Format.cs +++ b/libs/common/Format.cs @@ -128,7 +128,7 @@ public static async Task TryCreateEndpoint(string singleAddressOrHos } catch (Exception ex) { - logger?.LogError(ex, "Error while trying to resolve hostname:{hostname}", singleAddressOrHostname); + logger?.LogError("Error while trying to resolve hostname: {exMessage} [{hostname}]", ex.Message, singleAddressOrHostname); } return null; diff --git a/libs/host/Configuration/OptionsValidators.cs b/libs/host/Configuration/OptionsValidators.cs index 392cf367e1a..83d8e764c2c 100644 --- a/libs/host/Configuration/OptionsValidators.cs +++ b/libs/host/Configuration/OptionsValidators.cs @@ -361,8 +361,7 @@ protected override ValidationResult IsValid(object value, ValidationContext vali return initValidationResult; var logger = ((Options)validationContext.ObjectInstance).runtimeLogger; - - if (!Format.TryParseAddressList(ipAddresses, 0, out var endpoints, out var errorHostnameOrAddress, useForBind: false, logger: logger)) + if (!Format.TryParseAddressList(ipAddresses, 0, out _, out var errorHostnameOrAddress, useForBind: false, logger: logger)) { var baseError = validationContext.MemberName != null ? base.FormatErrorMessage(validationContext.MemberName) : string.Empty; var errorMessage = $"{baseError} Expected string in IPv4 / IPv6 format (e.g. 127.0.0.1 / 0:0:0:0:0:0:0:1) or 'localhost' or valid hostname. Actual value: {errorHostnameOrAddress}"; From 9f071837bcdf9e61a0854952c596c70320a95a22 Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Wed, 12 Mar 2025 10:45:31 -0700 Subject: [PATCH 11/26] allow unix and tcp socket use --- libs/host/Configuration/Options.cs | 15 ++++----------- libs/host/GarnetServer.cs | 13 +++++++------ 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/libs/host/Configuration/Options.cs b/libs/host/Configuration/Options.cs index c0290d68360..b47b5b06a8a 100644 --- a/libs/host/Configuration/Options.cs +++ b/libs/host/Configuration/Options.cs @@ -6,7 +6,6 @@ using System.ComponentModel.DataAnnotations; using System.IO; using System.Linq; -using System.Net; using System.Net.Sockets; using System.Reflection; using System.Runtime.InteropServices; @@ -655,17 +654,11 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null) var checkpointDir = CheckpointDir; if (!useAzureStorage) checkpointDir = new DirectoryInfo(string.IsNullOrEmpty(checkpointDir) ? (string.IsNullOrEmpty(logDir) ? "." : logDir) : checkpointDir).FullName; - EndPoint[] endpoints; + if (!Format.TryParseAddressList(Address, Port, out var endpoints, out _) || endpoints.Length == 0) + throw new GarnetException($"Invalid endpoint format {Address} {Port}."); + if (!string.IsNullOrEmpty(UnixSocketPath)) - { - endpoints = new EndPoint[1]; - endpoints[0] = new UnixDomainSocketEndPoint(UnixSocketPath); - } - else - { - if (!Format.TryParseAddressList(Address, Port, out endpoints, out _) || endpoints.Length == 0) - throw new GarnetException($"Invalid endpoint format {Address} {Port}."); - } + endpoints = [.. endpoints, new UnixDomainSocketEndPoint(UnixSocketPath)]; // Unix file permission octal to UnixFileMode var unixSocketPermissions = (UnixFileMode)Convert.ToInt32(UnixSocketPermission.ToString(), 8); diff --git a/libs/host/GarnetServer.cs b/libs/host/GarnetServer.cs index 4db0e7e1ef2..4850841acb2 100644 --- a/libs/host/GarnetServer.cs +++ b/libs/host/GarnetServer.cs @@ -248,18 +248,19 @@ private void InitializeServer() logger.LogInformation("Total configured memory limit: {configMemoryLimit}", configMemoryLimit); } - if (opts.EndPoints[0] is UnixDomainSocketEndPoint) - { - // Delete existing unix socket file, if it exists. - File.Delete(opts.UnixSocketPath); - } - // Create Garnet TCP server if none was provided. if (servers == null) { servers = new IGarnetServer[opts.EndPoints.Length]; for (var i = 0; i < servers.Length; i++) + { + if (opts.EndPoints[i] is UnixDomainSocketEndPoint) + { + // Delete existing unix socket file, if it exists. + File.Delete(opts.UnixSocketPath); + } servers[i] = new GarnetServerTcp(opts.EndPoints[i], 0, opts.TlsOptions, opts.NetworkSendThrottleMax, opts.NetworkConnectionLimit, opts.UnixSocketPath, opts.UnixSocketPermission, logger); + } } storeWrapper = new StoreWrapper(version, RedisProtocolVersion, servers, store, objectStore, objectStoreSizeTracker, From c61ef2f3e8b94d8292b370651088e6f5be32f388 Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Thu, 13 Mar 2025 15:20:44 -0700 Subject: [PATCH 12/26] calculate ip to advertise from endpoint list --- libs/server/StoreWrapper.cs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/libs/server/StoreWrapper.cs b/libs/server/StoreWrapper.cs index 4ff4c85aff0..63ee0f04d34 100644 --- a/libs/server/StoreWrapper.cs +++ b/libs/server/StoreWrapper.cs @@ -225,10 +225,20 @@ public StoreWrapper( /// public string GetIp() { - if (((GarnetServerTcp)TcpServer[0]).EndPoint is not IPEndPoint localEndpoint) - throw new NotImplementedException("Cluster mode for unix domain sockets has not been implemented"); + IPEndPoint localEndPoint = null; + foreach (var server in servers) + { + if (((GarnetServerTcp)server).EndPoint is IPEndPoint point) + { + localEndPoint = point; + break; + } + } + + if (localEndPoint == null) + throw new GarnetException("Cluster mode requires definition of at least one TCP socket!"); - if (localEndpoint.Address.Equals(IPAddress.Any)) + if (localEndPoint.Address.Equals(IPAddress.Any)) { using (Socket socket = new(AddressFamily.InterNetwork, SocketType.Dgram, 0)) { @@ -237,7 +247,7 @@ public string GetIp() return endPoint.Address.ToString(); } } - else if (localEndpoint.Address.Equals(IPAddress.IPv6Any)) + else if (localEndPoint.Address.Equals(IPAddress.IPv6Any)) { using (Socket socket = new(AddressFamily.InterNetworkV6, SocketType.Dgram, 0)) { @@ -246,7 +256,7 @@ public string GetIp() return endPoint.Address.ToString(); } } - return localEndpoint.Address.ToString(); + return localEndPoint.Address.ToString(); } internal FunctionsState CreateFunctionsState() From 85cd37545be94dc1f8543f5175471ea4e2cd3065 Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Thu, 13 Mar 2025 15:38:54 -0700 Subject: [PATCH 13/26] remove ip address logging --- libs/cluster/Server/Gossip.cs | 2 +- libs/cluster/Session/RespClusterBasicCommands.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/cluster/Server/Gossip.cs b/libs/cluster/Server/Gossip.cs index 35d34438a9d..be0c8079e77 100644 --- a/libs/cluster/Server/Gossip.cs +++ b/libs/cluster/Server/Gossip.cs @@ -164,7 +164,7 @@ public async Task TryMeetAsync(string address, int port, bool acquireLock = true var endpoints = await Format.TryCreateEndpoint(address, port, useForBind: true, logger: logger); if (endpoints == null) { - logger?.LogError("Could not parse endpoint {address} {port}", address, port); + logger?.LogError("Invalid CLUSTER MEET endpoint!"); } gsn = new GarnetServerNode(clusterProvider, endpoints[0], tlsOptions?.TlsClientOptions, logger: logger); created = true; diff --git a/libs/cluster/Session/RespClusterBasicCommands.cs b/libs/cluster/Session/RespClusterBasicCommands.cs index 2b15f3ea71a..4db65c4e243 100644 --- a/libs/cluster/Session/RespClusterBasicCommands.cs +++ b/libs/cluster/Session/RespClusterBasicCommands.cs @@ -168,7 +168,7 @@ private bool NetworkClusterMeet(out bool invalidParameters) return true; } - logger?.LogTrace("CLUSTER MEET {ipaddressStr} {port}", ipAddress, port); + logger?.LogTrace("CLUSTER MEET"); clusterProvider.clusterManager.RunMeetTask(ipAddress, port); while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) SendAndReset(); From 58da5e409702533efe2aaacfb32d2b2c1d433aee Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Tue, 18 Mar 2025 11:46:10 -0700 Subject: [PATCH 14/26] check that at least one endpoint is TCP for cluster --- libs/cluster/Server/ClusterManager.cs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/libs/cluster/Server/ClusterManager.cs b/libs/cluster/Server/ClusterManager.cs index bc26615bd0f..fdf275afba3 100644 --- a/libs/cluster/Server/ClusterManager.cs +++ b/libs/cluster/Server/ClusterManager.cs @@ -50,8 +50,18 @@ public unsafe ClusterManager(ClusterProvider clusterProvider, ILogger logger = n clusterConfigDevice = deviceFactory.Get(new FileDescriptor(directoryName: "", fileName: "nodes.conf")); pool = new(1, (int)clusterConfigDevice.SectorSize); - if (opts.EndPoints[0] is not IPEndPoint endpoint) - throw new NotImplementedException("Cluster mode for unix domain sockets has not been implemented."); + IPEndPoint endpoint = null; + foreach (var endPoint in opts.EndPoints) + { + if (endPoint is IPEndPoint _endpoint) + { + endpoint = _endpoint; + break; + } + } + + if (endpoint == null) + throw new GarnetException("No valid IPEndPoint found in endPoint list"); var address = clusterProvider.storeWrapper.GetIp(); From a03386b585991837a13842feb44f6a50d0ad3b00 Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Thu, 20 Mar 2025 14:09:04 -0700 Subject: [PATCH 15/26] first round of comments --- libs/common/Format.cs | 14 +++++++------- libs/host/Configuration/Options.cs | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/libs/common/Format.cs b/libs/common/Format.cs index c55d8e28866..2002a4f3f4e 100644 --- a/libs/common/Format.cs +++ b/libs/common/Format.cs @@ -36,12 +36,12 @@ public static class Format /// /// Parse address list string containing address separated by whitespace /// - /// - /// - /// - /// - /// - /// + /// Space separated string of IP addresses + /// Endpoint Port + /// List of endpoints generated from the input IPs + /// Output error if any + /// Differentiate validation between use for --bind parsing or CLUSTER MEET + /// Logger /// True if parse and address validation was successful, otherwise false public static bool TryParseAddressList(string addressList, int port, out EndPoint[] endpoints, out string errorHostnameOrAddress, bool useForBind = false, ILogger logger = null) { @@ -54,7 +54,7 @@ public static bool TryParseAddressList(string addressList, int port, out EndPoin return true; } - var addresses = addressList.Split(' '); + var addresses = addressList.Trim().Split(' '); var endpointList = new List(); // Validate addresses and create endpoints foreach (var singleAddressOrHostname in addresses) diff --git a/libs/host/Configuration/Options.cs b/libs/host/Configuration/Options.cs index b47b5b06a8a..f10a66607f1 100644 --- a/libs/host/Configuration/Options.cs +++ b/libs/host/Configuration/Options.cs @@ -39,7 +39,7 @@ internal sealed class Options public int Port { get; set; } [IpAddressValidation(false)] - [Option("bind", Required = false, HelpText = "IP address to bind server to (default: any)")] + [Option("bind", Required = false, HelpText = "Space separated string of IP addresses to bind server to (default: any)")] public string Address { get; set; } [MemorySizeValidation] From 87a5481cc7064974ce495579ce4eda8f2fad9183 Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Thu, 20 Mar 2025 17:27:32 -0700 Subject: [PATCH 16/26] remove empty entries --- libs/common/Format.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/common/Format.cs b/libs/common/Format.cs index 2002a4f3f4e..0765ed891cb 100644 --- a/libs/common/Format.cs +++ b/libs/common/Format.cs @@ -54,7 +54,7 @@ public static bool TryParseAddressList(string addressList, int port, out EndPoin return true; } - var addresses = addressList.Trim().Split(' '); + var addresses = addressList.Trim().Split(' ', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); var endpointList = new List(); // Validate addresses and create endpoints foreach (var singleAddressOrHostname in addresses) From 94902ff3b792193443134084aee0e88314dc36e2 Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Thu, 20 Mar 2025 17:33:50 -0700 Subject: [PATCH 17/26] add multiple socket test --- test/Garnet.test/GarnetClientTests.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test/Garnet.test/GarnetClientTests.cs b/test/Garnet.test/GarnetClientTests.cs index a00bd64fae3..f69389dafa2 100644 --- a/test/Garnet.test/GarnetClientTests.cs +++ b/test/Garnet.test/GarnetClientTests.cs @@ -3,6 +3,7 @@ using System; using System.Linq; +using System.Net; using System.Net.Sockets; using System.Text; using System.Threading; @@ -594,12 +595,13 @@ private static async Task DeleteKeysWithCT(string[] keys, Memory[] k } [Test] - public async Task UnixSocket_Ping([Values] bool useTls) + public async Task MultipleSocketPing([Values] bool useTls) { var unixSocketPath = "./unix-socket-ping-test.sock"; var unixSocketEndpoint = new UnixDomainSocketEndPoint(unixSocketPath); + var tcpEndpoint = new IPEndPoint(IPAddress.Loopback, TestUtils.TestPort); - using var server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, [unixSocketEndpoint], enableTLS: useTls, unixSocketPath: unixSocketPath); + using var server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, [unixSocketEndpoint, tcpEndpoint], enableTLS: useTls, unixSocketPath: unixSocketPath); server.Start(); using var db = TestUtils.GetGarnetClient(unixSocketEndpoint, useTLS: useTls); @@ -607,6 +609,12 @@ public async Task UnixSocket_Ping([Values] bool useTls) var result = await db.ExecuteForStringResultAsync("PING"); ClassicAssert.AreEqual("PONG", result); + + using var tcpClient = TestUtils.GetGarnetClient(tcpEndpoint, useTLS: useTls); + await tcpClient.ConnectAsync(); + + result = await db.ExecuteForStringResultAsync("PING"); + ClassicAssert.AreEqual("PONG", result); } } } \ No newline at end of file From cdc5ecb4243c06d917e2ffe536bb25a9f7cc25de Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Thu, 20 Mar 2025 17:47:25 -0700 Subject: [PATCH 18/26] nit: rename --- test/Garnet.test/GarnetServerConfigTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Garnet.test/GarnetServerConfigTests.cs b/test/Garnet.test/GarnetServerConfigTests.cs index 997b0e6ed88..24ef4618a35 100644 --- a/test/Garnet.test/GarnetServerConfigTests.cs +++ b/test/Garnet.test/GarnetServerConfigTests.cs @@ -721,7 +721,7 @@ public void UnixSocketPermission_InvalidPermissionFails() } [Test] - public async Task MultiEndpointTest() + public async Task MultiTcpSocketTest() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); var hostname = TestUtils.GetHostName(); From f86f10b1b3d8ccadd8f5a6b0461d1695f76b191e Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Thu, 20 Mar 2025 18:10:12 -0700 Subject: [PATCH 19/26] add better comments --- libs/cluster/Server/Gossip.cs | 2 +- libs/common/Format.cs | 19 +++++++++---------- libs/host/Configuration/OptionsValidators.cs | 2 +- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/libs/cluster/Server/Gossip.cs b/libs/cluster/Server/Gossip.cs index be0c8079e77..84078054091 100644 --- a/libs/cluster/Server/Gossip.cs +++ b/libs/cluster/Server/Gossip.cs @@ -161,7 +161,7 @@ public async Task TryMeetAsync(string address, int port, bool acquireLock = true if (gsn == null) { - var endpoints = await Format.TryCreateEndpoint(address, port, useForBind: true, logger: logger); + var endpoints = await Format.TryCreateEndpoint(address, port, tryConnect: true, logger: logger); if (endpoints == null) { logger?.LogError("Invalid CLUSTER MEET endpoint!"); diff --git a/libs/common/Format.cs b/libs/common/Format.cs index 0765ed891cb..e484cb5e1f5 100644 --- a/libs/common/Format.cs +++ b/libs/common/Format.cs @@ -40,10 +40,9 @@ public static class Format /// Endpoint Port /// List of endpoints generated from the input IPs /// Output error if any - /// Differentiate validation between use for --bind parsing or CLUSTER MEET /// Logger /// True if parse and address validation was successful, otherwise false - public static bool TryParseAddressList(string addressList, int port, out EndPoint[] endpoints, out string errorHostnameOrAddress, bool useForBind = false, ILogger logger = null) + public static bool TryParseAddressList(string addressList, int port, out EndPoint[] endpoints, out string errorHostnameOrAddress, ILogger logger = null) { endpoints = null; errorHostnameOrAddress = null; @@ -59,7 +58,7 @@ public static bool TryParseAddressList(string addressList, int port, out EndPoin // Validate addresses and create endpoints foreach (var singleAddressOrHostname in addresses) { - var e = TryCreateEndpoint(singleAddressOrHostname, port, useForBind, logger).Result; + var e = TryCreateEndpoint(singleAddressOrHostname, port, tryConnect: false, logger).Result; if(e == null) { endpoints = null; @@ -77,11 +76,11 @@ public static bool TryParseAddressList(string addressList, int port, out EndPoin /// Try to create an endpoint from address and port /// /// This could be an address or a hostname that the method tries to resolve - /// - /// Binding does not poll connection because is supposed to be called from the server side - /// + /// Port number to use for the endpoints + /// Whether to try to connect to the created endpoints to ensure that it is reachable + /// Logger /// - public static async Task TryCreateEndpoint(string singleAddressOrHostname, int port, bool useForBind = false, ILogger logger = null) + public static async Task TryCreateEndpoint(string singleAddressOrHostname, int port, bool tryConnect = false, ILogger logger = null) { if (string.IsNullOrEmpty(singleAddressOrHostname) || string.IsNullOrWhiteSpace(singleAddressOrHostname)) return [new IPEndPoint(IPAddress.Any, port)]; @@ -102,12 +101,12 @@ public static async Task TryCreateEndpoint(string singleAddressOrHos return null; } - if (useForBind) + if (tryConnect) { foreach (var entry in ipAddresses) { var endpoint = new IPEndPoint(entry, port); - var IsListening = await IsReachable(endpoint); + var IsListening = await TryConnect(endpoint); if (IsListening) return [endpoint]; } } @@ -133,7 +132,7 @@ public static async Task TryCreateEndpoint(string singleAddressOrHos return null; - async Task IsReachable(IPEndPoint endpoint) + async Task TryConnect(IPEndPoint endpoint) { using (var tcpClient = new TcpClient()) { diff --git a/libs/host/Configuration/OptionsValidators.cs b/libs/host/Configuration/OptionsValidators.cs index 83d8e764c2c..82f6a319fff 100644 --- a/libs/host/Configuration/OptionsValidators.cs +++ b/libs/host/Configuration/OptionsValidators.cs @@ -361,7 +361,7 @@ protected override ValidationResult IsValid(object value, ValidationContext vali return initValidationResult; var logger = ((Options)validationContext.ObjectInstance).runtimeLogger; - if (!Format.TryParseAddressList(ipAddresses, 0, out _, out var errorHostnameOrAddress, useForBind: false, logger: logger)) + if (!Format.TryParseAddressList(ipAddresses, 0, out _, out var errorHostnameOrAddress, logger: logger)) { var baseError = validationContext.MemberName != null ? base.FormatErrorMessage(validationContext.MemberName) : string.Empty; var errorMessage = $"{baseError} Expected string in IPv4 / IPv6 format (e.g. 127.0.0.1 / 0:0:0:0:0:0:0:1) or 'localhost' or valid hostname. Actual value: {errorHostnameOrAddress}"; From 92c559ee7ca348b8e575dfe88d5485af83f69264 Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Thu, 20 Mar 2025 18:14:37 -0700 Subject: [PATCH 20/26] handle dash in addresses from bind optio --- libs/common/Format.cs | 3 +++ libs/host/Configuration/Redis/RedisOptions.cs | 4 ++-- test/Garnet.test/GarnetServerConfigTests.cs | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/libs/common/Format.cs b/libs/common/Format.cs index e484cb5e1f5..13ca82798cc 100644 --- a/libs/common/Format.cs +++ b/libs/common/Format.cs @@ -85,6 +85,9 @@ public static async Task TryCreateEndpoint(string singleAddressOrHos if (string.IsNullOrEmpty(singleAddressOrHostname) || string.IsNullOrWhiteSpace(singleAddressOrHostname)) return [new IPEndPoint(IPAddress.Any, port)]; + if (singleAddressOrHostname[0] == '-') + singleAddressOrHostname = singleAddressOrHostname.Substring(1); + if (singleAddressOrHostname.Equals("localhost", StringComparison.CurrentCultureIgnoreCase)) return [new IPEndPoint(IPAddress.Loopback, port)]; diff --git a/libs/host/Configuration/Redis/RedisOptions.cs b/libs/host/Configuration/Redis/RedisOptions.cs index d3faa21e840..7355c1d3297 100644 --- a/libs/host/Configuration/Redis/RedisOptions.cs +++ b/libs/host/Configuration/Redis/RedisOptions.cs @@ -24,8 +24,8 @@ internal class RedisOptions Specify your passphrase in the tls-key-file-pass option (or via the cert-password command line argument), if applicable. Specify your subject name via the cert-subject-name command line argument, if applicable."; - [RedisOption("bind", nameof(Options.Address), BindWarning, typeof(ArrayToFirstItemTransformer))] - public Option Bind { get; set; } + [RedisOption("bind", nameof(Options.Address), BindWarning)] + public Option Bind { get; set; } [RedisOption("enable-debug-command", nameof(Options.EnableDebugCommand))] public Option EnableDebugCommand { get; set; } diff --git a/test/Garnet.test/GarnetServerConfigTests.cs b/test/Garnet.test/GarnetServerConfigTests.cs index 24ef4618a35..099298deae5 100644 --- a/test/Garnet.test/GarnetServerConfigTests.cs +++ b/test/Garnet.test/GarnetServerConfigTests.cs @@ -166,7 +166,7 @@ public void ImportExportRedisConfigLocal() var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, silentMode: true); ClassicAssert.IsTrue(parseSuccessful); ClassicAssert.AreEqual(invalidOptions.Count, 0); - ClassicAssert.AreEqual("127.0.0.1", options.Address); + ClassicAssert.AreEqual("127.0.0.1 -::1", options.Address); ClassicAssert.AreEqual(ConnectionProtectionOption.Local, options.EnableDebugCommand); ClassicAssert.AreEqual(6379, options.Port); ClassicAssert.AreEqual("20gb", options.MemorySize); From e0b570cf643ceb56b1f979e4e61e89126aced2c2 Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Fri, 21 Mar 2025 09:17:56 -0700 Subject: [PATCH 21/26] add test for checking parsing of multiple addresses --- test/Garnet.test/GarnetServerConfigTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Garnet.test/GarnetServerConfigTests.cs b/test/Garnet.test/GarnetServerConfigTests.cs index 099298deae5..8cefd329850 100644 --- a/test/Garnet.test/GarnetServerConfigTests.cs +++ b/test/Garnet.test/GarnetServerConfigTests.cs @@ -134,7 +134,7 @@ public void ImportExportConfigLocal() // No import path, include command line args // Check that all invalid options flagged - args = ["--bind", "1.1.1.257", "-m", "12mg", "--port", "-1", "--mutable-percent", "101", "--acl-file", "nx_dir/nx_file.txt", "--tls", "--reviv-fraction", "1.1", "--cert-file-name", "testcert.crt"]; + args = ["--bind", "1.1.1.257 127.0.0.1 -::1", "-m", "12mg", "--port", "-1", "--mutable-percent", "101", "--acl-file", "nx_dir/nx_file.txt", "--tls", "--reviv-fraction", "1.1", "--cert-file-name", "testcert.crt"]; parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out options, out invalidOptions, out exitGracefully, silentMode: true); ClassicAssert.IsFalse(parseSuccessful); ClassicAssert.IsFalse(exitGracefully); From 5f488f4b4ee7f83940d4f44e8c4161ada65ae730 Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Fri, 21 Mar 2025 10:23:43 -0700 Subject: [PATCH 22/26] remove unused code --- libs/common/Format.cs | 48 ------------------------------------------- 1 file changed, 48 deletions(-) diff --git a/libs/common/Format.cs b/libs/common/Format.cs index 13ca82798cc..578e96deed7 100644 --- a/libs/common/Format.cs +++ b/libs/common/Format.cs @@ -154,54 +154,6 @@ async Task TryConnect(IPEndPoint endpoint) } } - /// - /// Try to - /// - /// - /// - /// - /// - public static async Task TryValidateAndConnectAddress2(string address, int port, ILogger logger = null) - { - IPEndPoint endpoint = null; - if (!IPAddress.TryParse(address, out var ipAddress)) - { - // Try to identify reachable IP address from hostname - var hostEntry = Dns.GetHostEntry(address); - foreach (var entry in hostEntry.AddressList) - { - endpoint = new IPEndPoint(entry, port); - var IsListening = await IsReachable(endpoint); - if (IsListening) break; - } - } - else - { - // If address is valid create endpoint - endpoint = new IPEndPoint(ipAddress, port); - } - - async Task IsReachable(IPEndPoint endpoint) - { - using (var tcpClient = new TcpClient()) - { - try - { - await tcpClient.ConnectAsync(endpoint.Address, endpoint.Port); - logger?.LogTrace("Reachable {ip} {port}", endpoint.Address, endpoint.Port); - return true; - } - catch - { - logger?.LogTrace("Unreachable {ip} {port}", endpoint.Address, endpoint.Port); - return false; - } - } - } - - return endpoint; - } - /// /// Parse address (hostname) and port to endpoint /// From 073cebfa181dfa2231ac9e1da1af45a825aedca9 Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Tue, 25 Mar 2025 19:06:00 -0700 Subject: [PATCH 23/26] Fix cluster BDN and add ClusterAnnounce options --- .../BDN.benchmark/Cluster/ClusterContext.cs | 1 + .../Embedded/EmbeddedRespServer.cs | 2 +- libs/host/Configuration/Options.cs | 21 ++++++++++++++++++- libs/host/defaults.conf | 6 ++++++ libs/server/Servers/ServerOptions.cs | 5 +++++ libs/server/StoreWrapper.cs | 19 ++++++++++++----- 6 files changed, 47 insertions(+), 7 deletions(-) diff --git a/benchmark/BDN.benchmark/Cluster/ClusterContext.cs b/benchmark/BDN.benchmark/Cluster/ClusterContext.cs index 1b2bd245be8..e428ca60a0f 100644 --- a/benchmark/BDN.benchmark/Cluster/ClusterContext.cs +++ b/benchmark/BDN.benchmark/Cluster/ClusterContext.cs @@ -37,6 +37,7 @@ public void SetupSingleInstance(bool disableSlotVerification = false) EnableCluster = !disableSlotVerification, EndPoints = [new IPEndPoint(IPAddress.Loopback, port)], CleanClusterConfig = true, + ClusterAnnounceEndpoint = new IPEndPoint(IPAddress.Loopback, port) }; if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) opt.CheckpointDir = "/tmp"; diff --git a/benchmark/BDN.benchmark/Embedded/EmbeddedRespServer.cs b/benchmark/BDN.benchmark/Embedded/EmbeddedRespServer.cs index 3a0f7b2e0f7..5e571a09526 100644 --- a/benchmark/BDN.benchmark/Embedded/EmbeddedRespServer.cs +++ b/benchmark/BDN.benchmark/Embedded/EmbeddedRespServer.cs @@ -21,7 +21,7 @@ internal sealed class EmbeddedRespServer : GarnetServer /// Server options to configure the base GarnetServer instance /// Logger factory to configure the base GarnetServer instance /// Server network - public EmbeddedRespServer(GarnetServerOptions opts, ILoggerFactory loggerFactory = null, GarnetServerEmbedded server = null) : base(opts, loggerFactory, [server]) + public EmbeddedRespServer(GarnetServerOptions opts, ILoggerFactory loggerFactory = null, GarnetServerEmbedded server = null) : base(opts, loggerFactory, server == null ? null : [server]) { this.garnetServerEmbedded = server; this.subscribeBroker = opts.DisablePubSub ? null : diff --git a/libs/host/Configuration/Options.cs b/libs/host/Configuration/Options.cs index f10a66607f1..d4ef76b6f6b 100644 --- a/libs/host/Configuration/Options.cs +++ b/libs/host/Configuration/Options.cs @@ -6,6 +6,7 @@ using System.ComponentModel.DataAnnotations; using System.IO; using System.Linq; +using System.Net; using System.Net.Sockets; using System.Reflection; using System.Runtime.InteropServices; @@ -39,9 +40,17 @@ internal sealed class Options public int Port { get; set; } [IpAddressValidation(false)] - [Option("bind", Required = false, HelpText = "Space separated string of IP addresses to bind server to (default: any)")] + [Option("bind", Required = false, HelpText = "Whitespace separated string of IP addresses to bind server to (default: any)")] public string Address { get; set; } + [IntRangeValidation(0, 65535)] + [Option("cluster-announce-port", Required = false, HelpText = "Port that this node advertises to other nodes to connect to for gossiping.")] + public int ClusterAnnouncePort { get; set; } + + [IpAddressValidation(false)] + [Option("cluster-announce-ip", Required = false, HelpText = "IP address that this node advertises to other nodes to connect to for gossiping.")] + public string ClusterAnnounceIp { get; set; } + [MemorySizeValidation] [Option('m', "memory", Required = false, HelpText = "Total log memory used in bytes (rounds down to power of 2)")] public string MemorySize { get; set; } @@ -657,6 +666,15 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null) if (!Format.TryParseAddressList(Address, Port, out var endpoints, out _) || endpoints.Length == 0) throw new GarnetException($"Invalid endpoint format {Address} {Port}."); + EndPoint[] clusterAnnounceEndpoint = null; + if (ClusterAnnounceIp != null) + { + ClusterAnnouncePort = ClusterAnnouncePort == 0 ? Port : ClusterAnnouncePort; + clusterAnnounceEndpoint = Format.TryCreateEndpoint(ClusterAnnounceIp, ClusterAnnouncePort, tryConnect: false, logger: logger).GetAwaiter().GetResult(); + if (clusterAnnounceEndpoint == null || !endpoints.Any(endpoint => endpoint.Equals(clusterAnnounceEndpoint[0]))) + throw new GarnetException("Cluster announce endpoint does not match list of listen endpoints provided!"); + } + if (!string.IsNullOrEmpty(UnixSocketPath)) endpoints = [.. endpoints, new UnixDomainSocketEndPoint(UnixSocketPath)]; @@ -713,6 +731,7 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null) return new GarnetServerOptions(logger) { EndPoints = endpoints, + ClusterAnnounceEndpoint = clusterAnnounceEndpoint?[0], MemorySize = MemorySize, PageSize = PageSize, SegmentSize = SegmentSize, diff --git a/libs/host/defaults.conf b/libs/host/defaults.conf index cd1e1f4c95a..4c35c103f0b 100644 --- a/libs/host/defaults.conf +++ b/libs/host/defaults.conf @@ -9,6 +9,12 @@ /* IP address to bind server to (default: any) */ "Address" : null, + /* Port that this node advertises to other nodes to connect to for gossiping. */ + "ClusterAnnouncePort" : 0, + + /* IP address that this node advertises to other nodes to connect to for gossiping. */ + "ClusterAnnounceIp" : null, + /* Total log memory used in bytes (rounds down to power of 2) */ "MemorySize" : "16g", diff --git a/libs/server/Servers/ServerOptions.cs b/libs/server/Servers/ServerOptions.cs index bcf61ee1d72..eff66d75762 100644 --- a/libs/server/Servers/ServerOptions.cs +++ b/libs/server/Servers/ServerOptions.cs @@ -19,6 +19,11 @@ public class ServerOptions /// public EndPoint[] EndPoints { get; set; } = [new IPEndPoint(IPAddress.Loopback, 6379)]; + /// + /// Cluster announce Endpoint + /// + public EndPoint ClusterAnnounceEndpoint { get; set; } + /// /// Total log memory used in bytes (rounds down to power of 2). /// diff --git a/libs/server/StoreWrapper.cs b/libs/server/StoreWrapper.cs index 63ee0f04d34..9f05a56bbc9 100644 --- a/libs/server/StoreWrapper.cs +++ b/libs/server/StoreWrapper.cs @@ -226,17 +226,26 @@ public StoreWrapper( public string GetIp() { IPEndPoint localEndPoint = null; - foreach (var server in servers) + if (serverOptions.ClusterAnnounceEndpoint == null) { - if (((GarnetServerTcp)server).EndPoint is IPEndPoint point) + foreach (var server in servers) { - localEndPoint = point; - break; + if (((GarnetServerTcp)server).EndPoint is IPEndPoint point) + { + localEndPoint = point; + break; + } } } + else + { + if (serverOptions.ClusterAnnounceEndpoint is IPEndPoint point) + localEndPoint = point; + } + // Fail if we cannot advertise an endpoint for remote nodes to connect to if (localEndPoint == null) - throw new GarnetException("Cluster mode requires definition of at least one TCP socket!"); + throw new GarnetException("Cluster mode requires definition of at least one TCP socket through either the --bind or --cluster-announce-ip options!"); if (localEndPoint.Address.Equals(IPAddress.Any)) { From ae608cdfd7ac805c3ddc202f123b9ff11d8c88a9 Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Tue, 25 Mar 2025 20:16:48 -0700 Subject: [PATCH 24/26] sigh! add comma separator --- libs/common/Format.cs | 2 +- libs/host/Configuration/Options.cs | 2 +- libs/host/defaults.conf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/common/Format.cs b/libs/common/Format.cs index 578e96deed7..4df0a4308ef 100644 --- a/libs/common/Format.cs +++ b/libs/common/Format.cs @@ -53,7 +53,7 @@ public static bool TryParseAddressList(string addressList, int port, out EndPoin return true; } - var addresses = addressList.Trim().Split(' ', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); + var addresses = addressList.Trim().Split([',',' '], StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); var endpointList = new List(); // Validate addresses and create endpoints foreach (var singleAddressOrHostname in addresses) diff --git a/libs/host/Configuration/Options.cs b/libs/host/Configuration/Options.cs index d4ef76b6f6b..48032d2f397 100644 --- a/libs/host/Configuration/Options.cs +++ b/libs/host/Configuration/Options.cs @@ -40,7 +40,7 @@ internal sealed class Options public int Port { get; set; } [IpAddressValidation(false)] - [Option("bind", Required = false, HelpText = "Whitespace separated string of IP addresses to bind server to (default: any)")] + [Option("bind", Required = false, HelpText = "Whitespace or comma separated string of IP addresses to bind server to (default: any)")] public string Address { get; set; } [IntRangeValidation(0, 65535)] diff --git a/libs/host/defaults.conf b/libs/host/defaults.conf index 4c35c103f0b..1ceed823158 100644 --- a/libs/host/defaults.conf +++ b/libs/host/defaults.conf @@ -6,7 +6,7 @@ /* Port to run server on */ "Port" : 6379, - /* IP address to bind server to (default: any) */ + /* Whitespace or comma separated string of IP addresses to bind server to (default: any) */ "Address" : null, /* Port that this node advertises to other nodes to connect to for gossiping. */ From 97e7726cfb68632493d8d5fb0da7b2c9240b082c Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Wed, 26 Mar 2025 09:41:32 -0700 Subject: [PATCH 25/26] remove trim --- libs/common/Format.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/common/Format.cs b/libs/common/Format.cs index 4df0a4308ef..72a107321f6 100644 --- a/libs/common/Format.cs +++ b/libs/common/Format.cs @@ -53,7 +53,7 @@ public static bool TryParseAddressList(string addressList, int port, out EndPoin return true; } - var addresses = addressList.Trim().Split([',',' '], StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); + var addresses = addressList.Split([',',' '], StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); var endpointList = new List(); // Validate addresses and create endpoints foreach (var singleAddressOrHostname in addresses) From 7c0f47d87d13e277d81741a6bc08f2e24d540499 Mon Sep 17 00:00:00 2001 From: Vasileios Zois Date: Wed, 26 Mar 2025 14:09:40 -0700 Subject: [PATCH 26/26] log config ip --- libs/host/GarnetServer.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libs/host/GarnetServer.cs b/libs/host/GarnetServer.cs index 4850841acb2..e5564f5a7c3 100644 --- a/libs/host/GarnetServer.cs +++ b/libs/host/GarnetServer.cs @@ -185,7 +185,10 @@ private void InitializeServer() var clusterFactory = opts.EnableCluster ? new ClusterFactory() : null; this.logger = this.loggerFactory?.CreateLogger("GarnetServer"); - logger?.LogInformation("Garnet {version} {bits} bit; {clusterMode} mode; Endpoint: {endpoint}", version, IntPtr.Size == 8 ? "64" : "32", opts.EnableCluster ? "cluster" : "standalone", opts.EndPoints[0]); + logger?.LogInformation("Garnet {version} {bits} bit; {clusterMode} mode; Endpoint: [{endpoint}]", + version, IntPtr.Size == 8 ? "64" : "32", + opts.EnableCluster ? "cluster" : "standalone", + string.Join(',', opts.EndPoints.Select(endpoint => endpoint.ToString()))); logger?.LogInformation("Environment .NET {netVersion}; {osPlatform}; {processArch}", Environment.Version, Environment.OSVersion.Platform, RuntimeInformation.ProcessArchitecture); // Flush initialization logs from memory logger