Implement flag --experimental-set-member-localaddr

Which sets the LocalAddr to an IP address from --initial-advertise-peer-urls.

Also adds e2e test that requires this flag to succeed.

Signed-off-by: Edwin Xie <edwin.xie@broadcom.com>

CHANGELOG/CHANGELOG-3.6.md

@@ -75,6 +75,7 @@ See [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0...v3.6.0).
 - Decreased [`--snapshot-count` default value from 100,000 to 10,000](https://github.com/etcd-io/etcd/pull/15408)
 - Add [`etcd --tls-min-version --tls-max-version`](https://github.com/etcd-io/etcd/pull/15156) to enable support for TLS 1.3.
 - Add [quota to endpoint status response](https://github.com/etcd-io/etcd/pull/17877)
+- Add ['etcd --experimental-set-member-localaddr'](https://github.com/etcd-io/etcd/pull/17661) to enable using the first specified and non-loopback local address from initial-advertise-peer-urls as the local address when communicating with a peer.
 
 ### etcd grpc-proxy
 

client/pkg/transport/listener.go

@@ -194,7 +194,7 @@ type TLSInfo struct {
 	// If true, ClientConfig() will return an error for a cert with non empty CN.
 	EmptyCN bool
 
-	// LocalAddr is the local IP address to use when communicating peer.
+	// LocalAddr is the local IP address to use when communicating with a peer.
 	LocalAddr string
 }
 

server/config/config.go

@@ -205,8 +205,8 @@ type ServerConfig struct {
 	// V2Deprecation defines a phase of v2store deprecation process.
 	V2Deprecation V2DeprecationEnum `json:"v2-deprecation"`
 
-	// LocalAddr is the local IP address to use when communicating peer.
-	LocalAddr string `json:"local-address"`
+	// ExperimentalLocalAddress is the local IP address to use when communicating with a peer.
+	ExperimentalLocalAddress string `json:"experimental-local-address"`
 }
 
 // VerifyBootstrap sanity-checks the initial config for bootstrap case

server/embed/config.go

@@ -22,6 +22,7 @@ import (
 	"math"
 	"net"
 	"net/http"
+	"net/netip"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -225,6 +226,12 @@ type Config struct {
 	ClientAutoTLS bool
 	PeerTLSInfo   transport.TLSInfo
 	PeerAutoTLS   bool
+
+	// ExperimentalSetMemberLocalAddr enables using the first specified and
+	// non-loopback local address from initial-advertise-peer-urls as the local
+	// address when communicating with a peer.
+	ExperimentalSetMemberLocalAddr bool `json:"experimental-set-member-localaddr"`
+
 	// SelfSignedCertValidity specifies the validity period of the client and peer certificates
 	// that are automatically generated by etcd when you specify ClientAutoTLS and PeerAutoTLS,
 	// the unit is year, and the default is 1
@@ -335,9 +342,6 @@ type Config struct {
 	// AuthTokenTTL in seconds of the simple token
 	AuthTokenTTL uint `json:"auth-token-ttl"`
 
-	// PeerLocalAddr is the local IP address to use when communicating peer.
-	PeerLocalAddr string `json:"peer-local-addr"`
-
 	ExperimentalInitialCorruptCheck     bool          `json:"experimental-initial-corrupt-check"`
 	ExperimentalCorruptCheckTime        time.Duration `json:"experimental-corrupt-check-time"`
 	ExperimentalCompactHashCheckEnabled bool          `json:"experimental-compact-hash-check-enabled"`
@@ -624,6 +628,8 @@ func (cfg *Config) AddFlags(fs *flag.FlagSet) {
 		"initial-advertise-peer-urls",
 		"List of this member's peer URLs to advertise to the rest of the cluster.",
 	)
+	fs.BoolVar(&cfg.ExperimentalSetMemberLocalAddr, "experimental-set-member-localaddr", false, "Enable to have etcd use the first specified and non-loopback host from initial-advertise-peer-urls as the local address when communicating with a peer.")
+
 	fs.Var(
 		flags.NewUniqueURLsWithExceptions(DefaultAdvertiseClientURLs, ""),
 		"advertise-client-urls",
@@ -675,8 +681,6 @@ func (cfg *Config) AddFlags(fs *flag.FlagSet) {
 	fs.StringVar(&cfg.PeerTLSInfo.ClientKeyFile, "peer-client-key-file", "", "Path to an explicit peer client TLS key file otherwise peer key file will be used when client auth is required.")
 	fs.BoolVar(&cfg.PeerTLSInfo.ClientCertAuth, "peer-client-cert-auth", false, "Enable peer client cert authentication.")
 	fs.StringVar(&cfg.PeerTLSInfo.TrustedCAFile, "peer-trusted-ca-file", "", "Path to the peer server TLS trusted CA file.")
-	fs.StringVar(&cfg.PeerTLSInfo.LocalAddr, "peer-local-addr", "", "peer-local-addr is the local IP address to use when communicating peer.")
-
 	fs.BoolVar(&cfg.PeerAutoTLS, "peer-auto-tls", false, "Peer TLS using generated certificates")
 	fs.UintVar(&cfg.SelfSignedCertValidity, "self-signed-cert-validity", 1, "The validity period of the client and peer certificates, unit is year")
 	fs.StringVar(&cfg.PeerTLSInfo.CRLFile, "peer-crl-file", "", "Path to the peer certificate revocation list file.")
@@ -1153,6 +1157,40 @@ func (cfg *Config) InitialClusterFromName(name string) (ret string) {
 	return ret[1:]
 }
 
+// InferLocalAddr tries to determine the LocalAddr used when communicating with
+// an etcd peer. If SetMemberLocalAddr is true, then it will try to get the host
+// from AdvertisePeerUrls by searching for the first URL with a specified
+// non-loopback address. Otherwise, it defaults to empty string and the
+// LocalAddr used will be the default for the Golang HTTP client.
+func (cfg *Config) InferLocalAddr() string {
+	if !cfg.ExperimentalSetMemberLocalAddr {
+		return ""
+	}
+
+	lg := cfg.GetLogger()
+	lg.Info(
+		"searching for a suitable member local address in AdvertisePeerURLs",
+		zap.Strings("advertise-peer-urls", cfg.getAdvertisePeerURLs()),
+	)
+	for _, peerURL := range cfg.AdvertisePeerUrls {
+		if addr, err := netip.ParseAddr(peerURL.Hostname()); err == nil {
+			if addr.IsLoopback() || addr.IsUnspecified() {
+				continue
+			}
+			lg.Info(
+				"setting member local address",
+				zap.String("LocalAddr", addr.String()),
+			)
+			return addr.String()
+		}
+	}
+	lg.Warn(
+		"unable to set a member local address due to lack of suitable local addresses",
+		zap.Strings("advertise-peer-urls", cfg.getAdvertisePeerURLs()),
+	)
+	return ""
+}
+
 func (cfg *Config) IsNewCluster() bool { return cfg.ClusterState == ClusterStateFlagNew }
 func (cfg *Config) ElectionTicks() int { return int(cfg.ElectionMs / cfg.TickMs) }
 

server/embed/config_test.go

@@ -153,6 +153,138 @@ func TestUpdateDefaultClusterFromNameOverwrite(t *testing.T) {
 	}
 }
 
+func TestInferLocalAddr(t *testing.T) {
+	tests := []struct {
+		name               string
+		advertisePeerURLs  []string
+		setMemberLocalAddr bool
+		expectedLocalAddr  string
+	}{
+		{
+			"defaults, ExperimentalSetMemberLocalAddr=false ",
+			[]string{DefaultInitialAdvertisePeerURLs},
+			false,
+			"",
+		},
+		{
+			"IPv4 address, ExperimentalSetMemberLocalAddr=false ",
+			[]string{"https://192.168.100.110:2380"},
+			false,
+			"",
+		},
+		{
+			"defaults, ExperimentalSetMemberLocalAddr=true",
+			[]string{DefaultInitialAdvertisePeerURLs},
+			true,
+			"",
+		},
+		{
+			"IPv4 unspecified address, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://0.0.0.0:2380"},
+			true,
+			"",
+		},
+		{
+			"IPv6 unspecified address, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://[::]:2380"},
+			true,
+			"",
+		},
+		{
+			"IPv4 loopback address, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://127.0.0.1:2380"},
+			true,
+			"",
+		},
+		{
+			"IPv6 loopback address, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://[::1]:2380"},
+			true,
+			"",
+		},
+		{
+			"IPv4 address, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://192.168.100.110:2380"},
+			true,
+			"192.168.100.110",
+		},
+		{
+			"Hostname only, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://123-host-3.corp.internal:2380"},
+			true,
+			"",
+		},
+		{
+			"Hostname and IPv4 address, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://123-host-3.corp.internal:2380", "https://192.168.100.110:2380"},
+			true,
+			"192.168.100.110",
+		},
+		{
+			"IPv4 address and Hostname, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://192.168.100.110:2380", "https://123-host-3.corp.internal:2380"},
+			true,
+			"192.168.100.110",
+		},
+		{
+			"IPv4 and IPv6 addresses, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://192.168.100.110:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"},
+			true,
+			"192.168.100.110",
+		},
+		{
+			"IPv6 and IPv4 addresses, ExperimentalSetMemberLocalAddr=true",
+			// IPv4 addresses will always sort before IPv6 ones anyway
+			[]string{"https://[2001:db8:85a3::8a2e:370:7334]:2380", "https://192.168.100.110:2380"},
+			true,
+			"192.168.100.110",
+		},
+		{
+			"Hostname, IPv4 and IPv6 addresses, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://123-host-3.corp.internal:2380", "https://192.168.100.110:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"},
+			true,
+			"192.168.100.110",
+		},
+		{
+			"Hostname, IPv6 and IPv4 addresses, ExperimentalSetMemberLocalAddr=true",
+			// IPv4 addresses will always sort before IPv6 ones anyway
+			[]string{"https://123-host-3.corp.internal:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380", "https://192.168.100.110:2380"},
+			true,
+			"192.168.100.110",
+		},
+		{
+			"IPv6 address, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://[2001:db8:85a3::8a2e:370:7334]:2380"},
+			true,
+			"2001:db8:85a3::8a2e:370:7334",
+		},
+		{
+			"Hostname and IPv6 address, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://123-host-3.corp.internal:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"},
+			true,
+			"2001:db8:85a3::8a2e:370:7334",
+		},
+		{
+			"IPv6 address and Hostname, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://[2001:db8:85a3::8a2e:370:7334]:2380", "https://123-host-3.corp.internal:2380"},
+			true,
+			"2001:db8:85a3::8a2e:370:7334",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := NewConfig()
+			cfg.AdvertisePeerUrls = types.MustNewURLs(tt.advertisePeerURLs)
+			cfg.ExperimentalSetMemberLocalAddr = tt.setMemberLocalAddr
+
+			require.NoError(t, cfg.Validate())
+			require.Equal(t, tt.expectedLocalAddr, cfg.InferLocalAddr())
+		})
+	}
+
+}
+
 func (s *securityConfig) equals(t *transport.TLSInfo) bool {
 	return s.CertFile == t.CertFile &&
 		s.CertAuth == t.ClientCertAuth &&

server/embed/etcd.go

@@ -227,6 +227,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
 		ExperimentalBootstrapDefragThresholdMegabytes: cfg.ExperimentalBootstrapDefragThresholdMegabytes,
 		ExperimentalMaxLearners:                       cfg.ExperimentalMaxLearners,
 		V2Deprecation:                                 cfg.V2DeprecationEffective(),
+		ExperimentalLocalAddress:                      cfg.InferLocalAddr(),
 	}
 
 	if srvcfg.ExperimentalEnableDistributedTracing {
@@ -245,6 +246,8 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
 		)
 	}
 
+	srvcfg.PeerTLSInfo.LocalAddr = srvcfg.ExperimentalLocalAddress
+
 	print(e.cfg.logger, *cfg, srvcfg, memberInitialized)
 
 	if e.Server, err = etcdserver.NewServer(srvcfg); err != nil {
@@ -336,6 +339,8 @@ func print(lg *zap.Logger, ec Config, sc config.ServerConfig, memberInitialized
 		zap.Strings("advertise-client-urls", ec.getAdvertiseClientURLs()),
 		zap.Strings("listen-client-urls", ec.getListenClientURLs()),
 		zap.Strings("listen-metrics-urls", ec.getMetricsURLs()),
+		zap.Bool("experimental-set-member-localaddr", ec.ExperimentalSetMemberLocalAddr),
+		zap.String("experimental-local-address", sc.ExperimentalLocalAddress),
 		zap.Strings("cors", cors),
 		zap.Strings("host-whitelist", hss),
 		zap.String("initial-cluster", sc.InitialPeerURLsMap.String()),

server/etcdmain/help.go

@@ -107,6 +107,8 @@ Member:
 Clustering:
   --initial-advertise-peer-urls 'http://localhost:2380'
     List of this member's peer URLs to advertise to the rest of the cluster.
+  --experimental-set-member-localaddr 'false'
+    Enable using the first specified and non-loopback local address from initial-advertise-peer-urls as the local address when communicating with a peer.
   --initial-cluster 'default=http://localhost:2380'
     Initial cluster configuration for bootstrapping.
   --initial-cluster-state 'new'
@@ -221,9 +223,6 @@ Security:
     Minimum TLS version supported by etcd. Possible values: TLS1.2, TLS1.3.
   --tls-max-version ''
     Maximum TLS version supported by etcd. Possible values: TLS1.2, TLS1.3 (empty will be auto-populated by Go).
-  --peer-local-addr ''
-    LocalAddr is the local IP address to use when communicating peer.
-
 
 Auth:
   --auth-token 'simple'