fix: stabilize etcd join and promote sequences

There were two issues with using discovery service for join and promote:

* on join, that resulted in joining too fast which triggers race bugs in
  etcd cert generation (to be fixed as separate PR)
* on promote, Talos has to connect to non-learner member of the cluster
  which is somehow "automatic" with Kuberentes discovery, as it only
  lists `kube-apiserver` running which is up only when etcd on the same
  node is healthy. etcd client doesn't allow to avoid learner members,
  as even getting a member list from a learner doesn't work (to be fixed
  as a separate PR)

Signed-off-by: Andrey Smirnov <andrey.smirnov@talos-systems.com>
(cherry picked from commit 8a038d4)

Author: smira

internal/app/machined/pkg/system/services/etcd.go

@@ -337,7 +337,7 @@ func addMember(ctx context.Context, r runtime.Runtime, addrs []string, name stri
 		return nil, 0, fmt.Errorf("failed to generate etcd PKI: %w", err)
 	}
 
-	client, err := etcd.NewClientFromControlPlaneIPs(ctx, r.State().V1Alpha2().Resources())
+	client, err := etcd.NewClientFromControlPlaneIPsNoDiscovery(ctx, r.State().V1Alpha2().Resources())
 	if err != nil {
 		return nil, 0, err
 	}
@@ -681,11 +681,11 @@ func promoteMember(ctx context.Context, r runtime.Runtime, memberID uint64) erro
 	// try to promote a member until it succeeds (call might fail until the member catches up with the leader)
 	// promote member call will fail until member catches up with the master
 	return retry.Constant(10*time.Minute,
-		retry.WithUnits(10*time.Second),
+		retry.WithUnits(15*time.Second),
 		retry.WithJitter(time.Second),
 		retry.WithErrorLogging(true),
 	).RetryWithContext(ctx, func(ctx context.Context) error {
-		client, err := etcd.NewClientFromControlPlaneIPs(ctx, r.State().V1Alpha2().Resources())
+		client, err := etcd.NewClientFromControlPlaneIPsNoDiscovery(ctx, r.State().V1Alpha2().Resources())
 		if err != nil {
 			return retry.ExpectedError(err)
 		}

internal/pkg/etcd/etcd.go

@@ -9,6 +9,7 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"math/rand"
 	"os"
 	"time"
 
@@ -73,6 +74,18 @@ func NewLocalClient() (client *Client, err error) {
 // NewClientFromControlPlaneIPs initializes and returns an etcd client
 // configured to talk to all members.
 func NewClientFromControlPlaneIPs(ctx context.Context, resources state.State) (client *Client, err error) {
+	return newClientFromControlPlaneIPs(ctx, resources, "")
+}
+
+// NewClientFromControlPlaneIPsNoDiscovery initializes and returns an etcd client
+// configured to talk to all members, but ignores discovery service data.
+//
+// Note: this is a temporary workaround for etcd join issues which will be removed once backported to Talos 1.1.x.
+func NewClientFromControlPlaneIPsNoDiscovery(ctx context.Context, resources state.State) (client *Client, err error) {
+	return newClientFromControlPlaneIPs(ctx, resources, k8s.ControlPlaneDiscoveredEndpointsID)
+}
+
+func newClientFromControlPlaneIPs(ctx context.Context, resources state.State, ignoreEndpointID string) (client *Client, err error) {
 	endpointResources, err := resources.List(ctx, resource.NewMetadata(k8s.ControlPlaneNamespaceName, k8s.EndpointType, "", resource.VersionUndefined))
 	if err != nil {
 		return nil, fmt.Errorf("error getting endpoints resources: %w", err)
@@ -82,6 +95,10 @@ func NewClientFromControlPlaneIPs(ctx context.Context, resources state.State) (c
 
 	// merge all endpoints into a single list
 	for _, res := range endpointResources.Items {
+		if res.Metadata().ID() == ignoreEndpointID {
+			continue
+		}
+
 		endpointAddrs = endpointAddrs.Merge(res.(*k8s.Endpoint))
 	}
 
@@ -96,6 +113,9 @@ func NewClientFromControlPlaneIPs(ctx context.Context, resources state.State) (c
 		endpoints[i] = net.FormatAddress(endpoints[i]) + ":2379"
 	}
 
+	// Shuffle endpoints to establish random order on each call to avoid patterns based on sorted IP list.
+	rand.Shuffle(len(endpoints), func(i, j int) { endpoints[i], endpoints[j] = endpoints[j], endpoints[i] })
+
 	return NewClient(endpoints)
 }