Merge pull request kubernetes#67832 from fabriziopandini/kubeadm-conf…

…ig-APIEndpoint

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

kubeadm config add support for more than one APIEndpoint

**What this PR does / why we need it**:
This PR completes the changes in kubeadm for management of more than one control plane instances introducing the possibility to configure more than one APIEndpoints

**Which issue(s) this PR fixes** :
refs kubernetes/kubeadm#911, refs kubernetes/kubeadm#963

**Special notes for your reviewer**:
Depends on:
- [x] kubernetes#67830

**Release note**:
```release-note
kubeadm: The kubeadm configuration now support definition of more than one control plane instances with their own APIEndpoint. The APIEndpoint for the "bootstrap" control plane instance should be defined using `InitConfiguration.APIEndpoint`, while the APIEndpoints for additional control plane instances should be added using `JoinConfiguration.APIEndpoint`.  
```

/cc @kubernetes/sig-cluster-lifecycle-pr-reviews
/sig cluster-lifecycle
/area kubeadm
/kind api-change
/kind enhancement
/assign @luxas
/assign @timothysc
/cc @chuckha @rosti @neolit123 @liztio

cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go

@@ -45,25 +45,14 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 		func(obj *kubeadm.InitConfiguration, c fuzz.Continue) {
 			c.FuzzNoCustom(obj)
 			fuzzClusterConfig(&obj.ClusterConfiguration)
-			obj.BootstrapTokens = []kubeadm.BootstrapToken{
-				{
-					Token: &kubeadm.BootstrapTokenString{
-						ID:     "abcdef",
-						Secret: "abcdef0123456789",
-					},
-					TTL:    &metav1.Duration{Duration: 1 * time.Hour},
-					Usages: []string{"foo"},
-					Groups: []string{"foo"},
-				},
-			}
-			obj.NodeRegistration = kubeadm.NodeRegistrationOptions{
-				CRISocket: "foo",
-				Name:      "foo",
-				Taints:    []v1.Taint{},
-			}
+			fuzzBootstrapTokens(&obj.BootstrapTokens)
+			fuzzNodeRegistration(&obj.NodeRegistration)
+			fuzzAPIEndpoint(&obj.APIEndpoint)
 		},
 		func(obj *kubeadm.JoinConfiguration, c fuzz.Continue) {
 			c.FuzzNoCustom(obj)
+			fuzzNodeRegistration(&obj.NodeRegistration)
+			fuzzAPIEndpoint(&obj.APIEndpoint)
 			obj.CACertPath = "foo"
 			obj.DiscoveryFile = "foo"
 			obj.DiscoveryToken = "foo"
@@ -72,18 +61,37 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 			obj.TLSBootstrapToken = "foo"
 			obj.Token = "foo"
 			obj.ClusterName = "foo"
-			obj.NodeRegistration = kubeadm.NodeRegistrationOptions{
-				CRISocket: "foo",
-				Name:      "foo",
-			}
 		},
 	}
 }
 
+func fuzzBootstrapTokens(obj *[]kubeadm.BootstrapToken) {
+	obj = &[]kubeadm.BootstrapToken{
+		{
+			Token: &kubeadm.BootstrapTokenString{
+				ID:     "abcdef",
+				Secret: "abcdef0123456789",
+			},
+			TTL:    &metav1.Duration{Duration: 1 * time.Hour},
+			Usages: []string{"foo"},
+			Groups: []string{"foo"},
+		},
+	}
+}
+
+func fuzzNodeRegistration(obj *kubeadm.NodeRegistrationOptions) {
+	obj.CRISocket = "foo"
+	obj.Name = "foo"
+	obj.Taints = []v1.Taint{}
+}
+
+func fuzzAPIEndpoint(obj *kubeadm.APIEndpoint) {
+	obj.BindPort = 20
+	obj.AdvertiseAddress = "foo"
+}
+
 func fuzzClusterConfig(obj *kubeadm.ClusterConfiguration) {
 	obj.KubernetesVersion = "v10"
-	obj.API.BindPort = 20
-	obj.API.AdvertiseAddress = "foo"
 	obj.Networking.ServiceSubnet = "10.96.0.0/12"
 	obj.Networking.DNSDomain = "cluster.local"
 	obj.CertificatesDir = "foo"

cmd/kubeadm/app/apis/kubeadm/register.go

@@ -48,6 +48,7 @@ func addKnownTypes(scheme *runtime.Scheme) error {
 	scheme.AddKnownTypes(SchemeGroupVersion,
 		&InitConfiguration{},
 		&ClusterConfiguration{},
+		&ClusterStatus{},
 		&JoinConfiguration{},
 	)
 	return nil

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -47,6 +47,9 @@ type InitConfiguration struct {
 
 	// NodeRegistration holds fields that relate to registering the new master node to the cluster
 	NodeRegistration NodeRegistrationOptions
+
+	// APIEndpoint represents the endpoint of the instance of the API server to be deployed on this node.
+	APIEndpoint APIEndpoint
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -59,8 +62,6 @@ type ClusterConfiguration struct {
 	// +k8s:conversion-gen=false
 	ComponentConfigs ComponentConfigs
 
-	// API holds configuration for the k8s apiserver.
-	API API
 	// Etcd holds configuration for etcd.
 	Etcd Etcd
 
@@ -148,8 +149,20 @@ type ComponentConfigs struct {
 // the roundtrip is considered valid, as semi-static values are set and preserved during a roundtrip.
 func (cc ComponentConfigs) Fuzz(c fuzz.Continue) {}
 
-// API struct contains elements of API server address.
-type API struct {
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// ClusterStatus contains the cluster status. The ClusterStatus will be stored in the kubeadm-config
+// ConfigMap in the cluster, and then updated by kubeadm when additional control plane instance joins or leaves the cluster.
+type ClusterStatus struct {
+	metav1.TypeMeta
+
+	// APIEndpoints currently available in the cluster, one for each control plane/api server instance.
+	// The key of the map is the IP of the host's default interface
+	APIEndpoints map[string]APIEndpoint
+}
+
+// APIEndpoint struct contains elements of API server instance deployed on a node.
+type APIEndpoint struct {
 	// AdvertiseAddress sets the IP address for the API server to advertise.
 	AdvertiseAddress string
 
@@ -314,9 +327,8 @@ type JoinConfiguration struct {
 	// control plane instance.
 	ControlPlane bool
 
-	// AdvertiseAddress sets the IP address for the API server to advertise; the
-	// API server will be installed only on nodes hosting an additional control plane instance.
-	AdvertiseAddress string
+	// APIEndpoint represents the endpoint of the instance of the API server eventually to be deployed on this node.
+	APIEndpoint APIEndpoint
 
 	// FeatureGates enabled by the user.
 	FeatureGates map[string]bool

cmd/kubeadm/app/apis/kubeadm/v1alpha2/conversion.go

@@ -35,16 +35,23 @@ func Convert_v1alpha2_InitConfiguration_To_kubeadm_InitConfiguration(in *InitCon
 	if err := split_v1alpha2_InitConfiguration_into_kubeadm_ClusterConfiguration(in, &out.ClusterConfiguration, s); err != nil {
 		return err
 	}
+	if err := split_v1alpha2_InitConfiguration_into_kubeadm_APIEndpoint(in, &out.APIEndpoint, s); err != nil {
+		return err
+	}
+	return nil
+}
+
+func split_v1alpha2_InitConfiguration_into_kubeadm_APIEndpoint(in *InitConfiguration, out *kubeadm.APIEndpoint, s conversion.Scope) error {
+	out.AdvertiseAddress = in.API.AdvertiseAddress
+	out.BindPort = in.API.BindPort
+	// in.API.ControlPlaneEndpoint will be splitted into ClusterConfiguration
 	return nil
 }
 
 func split_v1alpha2_InitConfiguration_into_kubeadm_ClusterConfiguration(in *InitConfiguration, out *kubeadm.ClusterConfiguration, s conversion.Scope) error {
 	if err := split_v1alpha2_InitConfiguration_into_kubeadm_ComponentConfigs(in, &out.ComponentConfigs, s); err != nil {
 		return err
 	}
-	if err := Convert_v1alpha2_API_To_kubeadm_API(&in.API, &out.API, s); err != nil {
-		return err
-	}
 	if err := Convert_v1alpha2_Networking_To_kubeadm_Networking(&in.Networking, &out.Networking, s); err != nil {
 		return err
 	}
@@ -71,14 +78,6 @@ func split_v1alpha2_InitConfiguration_into_kubeadm_ClusterConfiguration(in *Init
 	return nil
 }
 
-func Convert_v1alpha2_API_To_kubeadm_API(in *API, out *kubeadm.API, s conversion.Scope) error {
-	if err := autoConvert_v1alpha2_API_To_kubeadm_API(in, out, s); err != nil {
-		return err
-	}
-	// in.ControlPlaneEndpoint is assigned outside this function
-	return nil
-}
-
 func split_v1alpha2_InitConfiguration_into_kubeadm_ComponentConfigs(in *InitConfiguration, out *kubeadm.ComponentConfigs, s conversion.Scope) error {
 	if in.KubeProxy.Config != nil {
 		if out.KubeProxy == nil {
@@ -101,23 +100,32 @@ func split_v1alpha2_InitConfiguration_into_kubeadm_ComponentConfigs(in *InitConf
 	return nil
 }
 
+func Convert_v1alpha2_JoinConfiguration_To_kubeadm_JoinConfiguration(in *JoinConfiguration, out *kubeadm.JoinConfiguration, s conversion.Scope) error {
+	if err := autoConvert_v1alpha2_JoinConfiguration_To_kubeadm_JoinConfiguration(in, out, s); err != nil {
+		return err
+	}
+	out.APIEndpoint.AdvertiseAddress = in.AdvertiseAddress
+	out.APIEndpoint.BindPort = in.BindPort
+	return nil
+}
+
 func Convert_kubeadm_InitConfiguration_To_v1alpha2_InitConfiguration(in *kubeadm.InitConfiguration, out *InitConfiguration, s conversion.Scope) error {
 	if err := autoConvert_kubeadm_InitConfiguration_To_v1alpha2_InitConfiguration(in, out, s); err != nil {
 		return err
 	}
 	if err := join_kubeadm_ClusterConfiguration_into_v1alpha2_InitConfiguration(&in.ClusterConfiguration, out, s); err != nil {
 		return err
 	}
+	if err := join_kubeadm_APIEndpoint_into_v1alpha2_InitConfiguration(&in.APIEndpoint, out, s); err != nil {
+		return err
+	}
 	return nil
 }
 
 func join_kubeadm_ClusterConfiguration_into_v1alpha2_InitConfiguration(in *kubeadm.ClusterConfiguration, out *InitConfiguration, s conversion.Scope) error {
 	if err := join_kubeadm_ComponentConfigs_into_v1alpha2_InitConfiguration(&in.ComponentConfigs, out, s); err != nil {
 		return err
 	}
-	if err := Convert_kubeadm_API_To_v1alpha2_API(&in.API, &out.API, s); err != nil {
-		return err
-	}
 	if err := Convert_kubeadm_Etcd_To_v1alpha2_Etcd(&in.Etcd, &out.Etcd, s); err != nil {
 		return err
 	}
@@ -144,11 +152,10 @@ func join_kubeadm_ClusterConfiguration_into_v1alpha2_InitConfiguration(in *kubea
 	return nil
 }
 
-func Convert_kubeadm_API_To_v1alpha2_API(in *kubeadm.API, out *API, s conversion.Scope) error {
-	if err := autoConvert_kubeadm_API_To_v1alpha2_API(in, out, s); err != nil {
-		return err
-	}
-	// out.ControlPlaneEndpoint is assigned outside this function
+func join_kubeadm_APIEndpoint_into_v1alpha2_InitConfiguration(in *kubeadm.APIEndpoint, out *InitConfiguration, s conversion.Scope) error {
+	out.API.AdvertiseAddress = in.AdvertiseAddress
+	out.API.BindPort = in.BindPort
+	// out.API.ControlPlaneEndpoint will join from ClusterConfiguration
 	return nil
 }
 
@@ -173,3 +180,12 @@ func join_kubeadm_ComponentConfigs_into_v1alpha2_InitConfiguration(in *kubeadm.C
 	}
 	return nil
 }
+
+func Convert_kubeadm_JoinConfiguration_To_v1alpha2_JoinConfiguration(in *kubeadm.JoinConfiguration, out *JoinConfiguration, s conversion.Scope) error {
+	if err := autoConvert_kubeadm_JoinConfiguration_To_v1alpha2_JoinConfiguration(in, out, s); err != nil {
+		return err
+	}
+	out.AdvertiseAddress = in.APIEndpoint.AdvertiseAddress
+	out.BindPort = in.APIEndpoint.BindPort
+	return nil
+}

cmd/kubeadm/app/apis/kubeadm/v1alpha2/defaults.go

@@ -170,6 +170,10 @@ func SetDefaults_JoinConfiguration(obj *JoinConfiguration) {
 		obj.ClusterName = DefaultClusterName
 	}
 
+	if obj.BindPort == 0 {
+		obj.BindPort = DefaultAPIBindPort
+	}
+
 	SetDefaults_NodeRegistrationOptions(&obj.NodeRegistration)
 }
 

cmd/kubeadm/app/apis/kubeadm/v1alpha2/types.go

@@ -285,6 +285,10 @@ type JoinConfiguration struct {
 	// API server will be installed only on nodes hosting an additional control plane instance.
 	AdvertiseAddress string `json:"advertiseAddress,omitempty"`
 
+	// BindPort sets the secure port for the API Server to bind to.
+	// Defaults to 6443.
+	BindPort int32 `json:"bindPort,omitempty"`
+
 	// FeatureGates enabled by the user.
 	FeatureGates map[string]bool `json:"featureGates,omitempty"`
 }