kubeadm: deprecate the ClusterStatus dependency

cmd/kubeadm/app/apis/kubeadm/BUILD

@@ -9,6 +9,7 @@ load(
 go_library(
     name = "go_default_library",
     srcs = [
+        "apiendpoint.go",
         "bootstraptokenhelpers.go",
         "bootstraptokenstring.go",
         "doc.go",
@@ -52,6 +53,7 @@ filegroup(
 go_test(
     name = "go_default_test",
     srcs = [
+        "apiendpoint_test.go",
         "bootstraptokenhelpers_test.go",
         "bootstraptokenstring_test.go",
     ],

cmd/kubeadm/app/apis/kubeadm/apiendpoint.go

@@ -0,0 +1,44 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubeadm
+
+import (
+	"net"
+	"strconv"
+
+	"github.com/pkg/errors"
+)
+
+// APIEndpointFromString returns an APIEndpoint struct based on a "host:port" raw string.
+func APIEndpointFromString(apiEndpoint string) (APIEndpoint, error) {
+	apiEndpointHost, apiEndpointPortStr, err := net.SplitHostPort(apiEndpoint)
+	if err != nil {
+		return APIEndpoint{}, errors.Wrapf(err, "invalid advertise address endpoint: %s", apiEndpoint)
+	}
+	apiEndpointPort, err := net.LookupPort("tcp", apiEndpointPortStr)
+	if err != nil {
+		return APIEndpoint{}, errors.Wrapf(err, "invalid advertise address endpoint port: %s", apiEndpointPortStr)
+	}
+	return APIEndpoint{
+		AdvertiseAddress: apiEndpointHost,
+		BindPort:         int32(apiEndpointPort),
+	}, nil
+}
+
+func (endpoint *APIEndpoint) String() string {
+	return net.JoinHostPort(endpoint.AdvertiseAddress, strconv.FormatInt(int64(endpoint.BindPort), 10))
+}

cmd/kubeadm/app/apis/kubeadm/apiendpoint_test.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubeadm
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestAPIEndpointFromString(t *testing.T) {
+	var tests = []struct {
+		apiEndpoint      string
+		expectedEndpoint APIEndpoint
+		expectedErr      bool
+	}{
+		{apiEndpoint: "1.2.3.4:1234", expectedEndpoint: APIEndpoint{AdvertiseAddress: "1.2.3.4", BindPort: 1234}},
+		{apiEndpoint: "1.2.3.4:-1", expectedErr: true},
+		{apiEndpoint: "1.2.::1234", expectedErr: true},
+		{apiEndpoint: "1.2.3.4:65536", expectedErr: true},
+		{apiEndpoint: "[::1]:1234", expectedEndpoint: APIEndpoint{AdvertiseAddress: "::1", BindPort: 1234}},
+		{apiEndpoint: "[::1]:-1", expectedErr: true},
+		{apiEndpoint: "[::1]:65536", expectedErr: true},
+		{apiEndpoint: "[::1:1234", expectedErr: true},
+	}
+	for _, rt := range tests {
+		t.Run(rt.apiEndpoint, func(t *testing.T) {
+			apiEndpoint, err := APIEndpointFromString(rt.apiEndpoint)
+			if (err != nil) != rt.expectedErr {
+				t.Errorf("expected error %v, got %v, error: %v", rt.expectedErr, err != nil, err)
+			}
+			if !reflect.DeepEqual(apiEndpoint, rt.expectedEndpoint) {
+				t.Errorf("expected API endpoint: %v; got: %v", rt.expectedEndpoint, apiEndpoint)
+			}
+		})
+	}
+}

cmd/kubeadm/app/constants/BUILD

@@ -18,6 +18,7 @@ go_library(
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library",
         "//vendor/github.com/pkg/errors:go_default_library",
         "//vendor/k8s.io/utils/net:go_default_library",

cmd/kubeadm/app/constants/constants.go

@@ -29,6 +29,7 @@ import (
 	"github.com/pkg/errors"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/version"
+	"k8s.io/apimachinery/pkg/util/wait"
 	bootstrapapi "k8s.io/cluster-bootstrap/token/api"
 	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	utilnet "k8s.io/utils/net"
@@ -370,6 +371,16 @@ const (
 	// May be overridden by a flag at startup.
 	KubeControllerManagerPort = 10257
 
+	// EtcdAdvertiseClientUrlsAnnotationKey is the annotation key on every etcd pod, describing the
+	// advertise client URLs
+	EtcdAdvertiseClientUrlsAnnotationKey = "kubeadm.kubernetes.io/etcd.advertise-client-urls"
+	// KubeAPIServerAdvertiseAddressEndpointAnnotationKey is the annotation key on every apiserver pod,
+	// describing the API endpoint (advertise address and bind port of the api server)
+	KubeAPIServerAdvertiseAddressEndpointAnnotationKey = "kubeadm.kubernetes.io/kube-apiserver.advertise-address.endpoint"
+
+	// ControlPlaneTier is the value used in the tier label to identify control plane components
+	ControlPlaneTier = "control-plane"
+
 	// Mode* constants were copied from pkg/kubeapiserver/authorizer/modes
 	// to avoid kubeadm dependency on the internal module
 	// TODO: share Mode* constants in component config
@@ -432,6 +443,15 @@ var (
 	// KubeadmCertsClusterRoleName sets the name for the ClusterRole that allows
 	// the bootstrap tokens to access the kubeadm-certs Secret during the join of a new control-plane
 	KubeadmCertsClusterRoleName = fmt.Sprintf("kubeadm:%s", KubeadmCertsSecret)
+
+	// StaticPodMirroringDefaultRetry is used a backoff strategy for
+	// waiting for static pods to be mirrored to the apiserver.
+	StaticPodMirroringDefaultRetry = wait.Backoff{
+		Steps:    30,
+		Duration: 1 * time.Second,
+		Factor:   1.0,
+		Jitter:   0.1,
+	}
 )
 
 // EtcdSupportedVersion returns officially supported version of etcd for a specific Kubernetes release

cmd/kubeadm/app/phases/controlplane/manifests.go

@@ -59,7 +59,8 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
 			LivenessProbe:   staticpodutil.LivenessProbe(staticpodutil.GetAPIServerProbeAddress(endpoint), "/healthz", int(endpoint.BindPort), v1.URISchemeHTTPS),
 			Resources:       staticpodutil.ComponentResources("250m"),
 			Env:             kubeadmutil.GetProxyEnvVars(),
-		}, mounts.GetVolumes(kubeadmconstants.KubeAPIServer)),
+		}, mounts.GetVolumes(kubeadmconstants.KubeAPIServer),
+			map[string]string{kubeadmconstants.KubeAPIServerAdvertiseAddressEndpointAnnotationKey: endpoint.String()}),
 		kubeadmconstants.KubeControllerManager: staticpodutil.ComponentPod(v1.Container{
 			Name:            kubeadmconstants.KubeControllerManager,
 			Image:           images.GetKubernetesImage(kubeadmconstants.KubeControllerManager, cfg),
@@ -69,7 +70,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
 			LivenessProbe:   staticpodutil.LivenessProbe(staticpodutil.GetControllerManagerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeControllerManagerPort, v1.URISchemeHTTPS),
 			Resources:       staticpodutil.ComponentResources("200m"),
 			Env:             kubeadmutil.GetProxyEnvVars(),
-		}, mounts.GetVolumes(kubeadmconstants.KubeControllerManager)),
+		}, mounts.GetVolumes(kubeadmconstants.KubeControllerManager), nil),
 		kubeadmconstants.KubeScheduler: staticpodutil.ComponentPod(v1.Container{
 			Name:            kubeadmconstants.KubeScheduler,
 			Image:           images.GetKubernetesImage(kubeadmconstants.KubeScheduler, cfg),
@@ -79,7 +80,7 @@ func GetStaticPodSpecs(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmap
 			LivenessProbe:   staticpodutil.LivenessProbe(staticpodutil.GetSchedulerProbeAddress(cfg), "/healthz", kubeadmconstants.KubeSchedulerPort, v1.URISchemeHTTPS),
 			Resources:       staticpodutil.ComponentResources("100m"),
 			Env:             kubeadmutil.GetProxyEnvVars(),
-		}, mounts.GetVolumes(kubeadmconstants.KubeScheduler)),
+		}, mounts.GetVolumes(kubeadmconstants.KubeScheduler), nil),
 	}
 	return staticPodSpecs
 }

cmd/kubeadm/app/phases/etcd/local.go

@@ -181,18 +181,23 @@ func GetEtcdPodSpec(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.A
 	}
 	// probeHostname returns the correct localhost IP address family based on the endpoint AdvertiseAddress
 	probeHostname, probePort, probeScheme := staticpodutil.GetEtcdProbeEndpoint(&cfg.Etcd, utilsnet.IsIPv6String(endpoint.AdvertiseAddress))
-	return staticpodutil.ComponentPod(v1.Container{
-		Name:            kubeadmconstants.Etcd,
-		Command:         getEtcdCommand(cfg, endpoint, nodeName, initialCluster),
-		Image:           images.GetEtcdImage(cfg),
-		ImagePullPolicy: v1.PullIfNotPresent,
-		// Mount the etcd datadir path read-write so etcd can store data in a more persistent manner
-		VolumeMounts: []v1.VolumeMount{
-			staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.Local.DataDir, false),
-			staticpodutil.NewVolumeMount(certsVolumeName, cfg.CertificatesDir+"/etcd", false),
+	return staticpodutil.ComponentPod(
+		v1.Container{
+			Name:            kubeadmconstants.Etcd,
+			Command:         getEtcdCommand(cfg, endpoint, nodeName, initialCluster),
+			Image:           images.GetEtcdImage(cfg),
+			ImagePullPolicy: v1.PullIfNotPresent,
+			// Mount the etcd datadir path read-write so etcd can store data in a more persistent manner
+			VolumeMounts: []v1.VolumeMount{
+				staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.Local.DataDir, false),
+				staticpodutil.NewVolumeMount(certsVolumeName, cfg.CertificatesDir+"/etcd", false),
+			},
+			LivenessProbe: staticpodutil.LivenessProbe(probeHostname, "/health", probePort, probeScheme),
 		},
-		LivenessProbe: staticpodutil.LivenessProbe(probeHostname, "/health", probePort, probeScheme),
-	}, etcdMounts)
+		etcdMounts,
+		// etcd will listen on the advertise address of the API server, in a different port (2379)
+		map[string]string{kubeadmconstants.EtcdAdvertiseClientUrlsAnnotationKey: etcdutil.GetClientURL(endpoint)},
+	)
 }
 
 // getEtcdCommand builds the right etcd command from the given config object

cmd/kubeadm/app/util/config/BUILD

@@ -31,8 +31,10 @@ go_library(
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
         "//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library",
         "//staging/src/k8s.io/client-go/util/cert:go_default_library",
@@ -60,14 +62,18 @@ go_test(
         "//cmd/kubeadm/app/componentconfigs:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
         "//cmd/kubeadm/app/util:go_default_library",
-        "//cmd/kubeadm/app/util/apiclient:go_default_library",
+        "//cmd/kubeadm/test/resources:go_default_library",
         "//staging/src/k8s.io/api/core/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library",
-        "//staging/src/k8s.io/client-go/kubernetes:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
+        "//staging/src/k8s.io/client-go/testing:go_default_library",
         "//vendor/github.com/lithammer/dedent:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
         "//vendor/github.com/pmezard/go-difflib/difflib:go_default_library",
         "//vendor/sigs.k8s.io/yaml:go_default_library",
     ],