Merge pull request #67441 from rosti/kubeadm_clusterconfig

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

[reissue] kubeadm: Split out ClusterConfiguration from InitConfiguration

As @luxas is not able to take care of #66219, I am reissuing the same change here. There are a few minor things added by me:

- The original PR is rebased on latest master.
- Some broken tests were fixed.
- Some TODOs were added.
- Run update-bazel and update-gofmt

Below is the text of the original PR by Lucas.

-----

**What this PR does / why we need it:**

Splits MasterConfiguration to InitConfiguration and ClusterConfiguration as outlined in the kubeadm Config KEP. InitConfiguration holds init-only information, and ClusterConfiguration holds cluster-wide information. In the internal representation InitConfiguration wraps ClusterConfiguration as a field, but in serialized format they're different YAML documents.

**Which issue(s) this PR fixes** (optional, in fixes #<issue number>(, fixes #<issue_number>, ...) format, will close the issue(s) when PR gets merged):
ref: kubernetes/kubeadm#911
Depends on:

- [X] #65776
- [X] #65628
- [X] #65629
- [X] #65631
- [X] #65940
- [X] #65787
- [X] #65945
- [X] #65951
- [X] #65952

**Special notes for your reviewer:**

**Release note**:
```release-note
kubeadm: InitConfiguration now consists of two structs: InitConfiguration and ClusterConfiguration
```

@kubernetes/sig-cluster-lifecycle-pr-reviews

cmd/kubeadm/app/apis/kubeadm/fuzzer/BUILD

@@ -42,8 +42,4 @@ go_test(
     name = "go_default_test",
     srcs = ["fuzzer_test.go"],
     embed = [":go_default_library"],
-    deps = [
-        "//cmd/kubeadm/app/apis/kubeadm/scheme:go_default_library",
-        "//staging/src/k8s.io/apimachinery/pkg/api/apitesting/roundtrip:go_default_library",
-    ],
 )

cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go

@@ -33,18 +33,18 @@ import (
 	utilpointer "k8s.io/utils/pointer"
 )
 
+// NOTE: Right now this code is unused, as the test utilizing this is disabled.
+
 // Funcs returns the fuzzer functions for the kubeadm apis.
 func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 	return []interface{}{
+		func(obj *kubeadm.ClusterConfiguration, c fuzz.Continue) {
+			c.FuzzNoCustom(obj)
+			fuzzClusterConfig(obj)
+		},
 		func(obj *kubeadm.InitConfiguration, c fuzz.Continue) {
 			c.FuzzNoCustom(obj)
-			obj.KubernetesVersion = "v10"
-			obj.API.BindPort = 20
-			obj.API.AdvertiseAddress = "foo"
-			obj.Networking.ServiceSubnet = "10.96.0.0/12"
-			obj.Networking.DNSDomain = "cluster.local"
-			obj.CertificatesDir = "foo"
-			obj.APIServerCertSANs = []string{"foo"}
+			fuzzClusterConfig(&obj.ClusterConfiguration)
 			obj.BootstrapTokens = []kubeadm.BootstrapToken{
 				{
 					Token: &kubeadm.BootstrapTokenString{
@@ -56,47 +56,11 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 					Groups: []string{"foo"},
 				},
 			}
-			obj.ImageRepository = "foo"
-			obj.CIImageRepository = ""
-			obj.UnifiedControlPlaneImage = "foo"
-			obj.FeatureGates = map[string]bool{"foo": true}
-			obj.ClusterName = "foo"
-			obj.APIServerExtraArgs = map[string]string{"foo": "foo"}
-			obj.APIServerExtraVolumes = []kubeadm.HostPathMount{{
-				Name:      "foo",
-				HostPath:  "foo",
-				MountPath: "foo",
-				Writable:  false,
-			}}
-			obj.Etcd.Local = &kubeadm.LocalEtcd{
-				Image:          "foo",
-				DataDir:        "foo",
-				ServerCertSANs: []string{"foo"},
-				PeerCertSANs:   []string{"foo"},
-				ExtraArgs:      map[string]string{"foo": "foo"},
-			}
 			obj.NodeRegistration = kubeadm.NodeRegistrationOptions{
 				CRISocket: "foo",
 				Name:      "foo",
 				Taints:    []v1.Taint{},
 			}
-			obj.AuditPolicyConfiguration = kubeadm.AuditPolicyConfiguration{
-				Path:      "foo",
-				LogDir:    "/foo",
-				LogMaxAge: utilpointer.Int32Ptr(0),
-			}
-			// Set the Kubelet ComponentConfig to an empty, defaulted struct
-			extkubeletconfig := &kubeletconfigv1beta1.KubeletConfiguration{}
-			obj.ComponentConfigs.Kubelet = &kubeletconfig.KubeletConfiguration{}
-			componentconfigs.Scheme.Default(extkubeletconfig)
-			componentconfigs.Scheme.Convert(extkubeletconfig, obj.ComponentConfigs.Kubelet, nil)
-			componentconfigs.DefaultKubeletConfiguration(obj)
-			// Set the KubeProxy ComponentConfig to an empty, defaulted struct
-			extkubeproxyconfig := &kubeproxyconfigv1alpha1.KubeProxyConfiguration{}
-			obj.ComponentConfigs.KubeProxy = &kubeproxyconfig.KubeProxyConfiguration{}
-			componentconfigs.Scheme.Default(extkubeproxyconfig)
-			componentconfigs.Scheme.Convert(extkubeproxyconfig, obj.ComponentConfigs.KubeProxy, nil)
-			componentconfigs.DefaultKubeProxyConfiguration(obj)
 		},
 		func(obj *kubeadm.JoinConfiguration, c fuzz.Continue) {
 			c.FuzzNoCustom(obj)
@@ -115,3 +79,49 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 		},
 	}
 }
+
+func fuzzClusterConfig(obj *kubeadm.ClusterConfiguration) {
+	obj.KubernetesVersion = "v10"
+	obj.API.BindPort = 20
+	obj.API.AdvertiseAddress = "foo"
+	obj.Networking.ServiceSubnet = "10.96.0.0/12"
+	obj.Networking.DNSDomain = "cluster.local"
+	obj.CertificatesDir = "foo"
+	obj.APIServerCertSANs = []string{"foo"}
+	obj.ImageRepository = "foo"
+	obj.CIImageRepository = ""
+	obj.UnifiedControlPlaneImage = "foo"
+	obj.FeatureGates = map[string]bool{"foo": true}
+	obj.ClusterName = "foo"
+	obj.APIServerExtraArgs = map[string]string{"foo": "foo"}
+	obj.APIServerExtraVolumes = []kubeadm.HostPathMount{{
+		Name:      "foo",
+		HostPath:  "foo",
+		MountPath: "foo",
+		Writable:  false,
+	}}
+	obj.Etcd.Local = &kubeadm.LocalEtcd{
+		Image:          "foo",
+		DataDir:        "foo",
+		ServerCertSANs: []string{"foo"},
+		PeerCertSANs:   []string{"foo"},
+		ExtraArgs:      map[string]string{"foo": "foo"},
+	}
+	obj.AuditPolicyConfiguration = kubeadm.AuditPolicyConfiguration{
+		Path:      "foo",
+		LogDir:    "/foo",
+		LogMaxAge: utilpointer.Int32Ptr(0),
+	}
+	// Set the Kubelet ComponentConfig to an empty, defaulted struct
+	extkubeletconfig := &kubeletconfigv1beta1.KubeletConfiguration{}
+	obj.ComponentConfigs.Kubelet = &kubeletconfig.KubeletConfiguration{}
+	componentconfigs.Scheme.Default(extkubeletconfig)
+	componentconfigs.Scheme.Convert(extkubeletconfig, obj.ComponentConfigs.Kubelet, nil)
+	componentconfigs.DefaultKubeletConfiguration(obj)
+	// Set the KubeProxy ComponentConfig to an empty, defaulted struct
+	extkubeproxyconfig := &kubeproxyconfigv1alpha1.KubeProxyConfiguration{}
+	obj.ComponentConfigs.KubeProxy = &kubeproxyconfig.KubeProxyConfiguration{}
+	componentconfigs.Scheme.Default(extkubeproxyconfig)
+	componentconfigs.Scheme.Convert(extkubeproxyconfig, obj.ComponentConfigs.KubeProxy, nil)
+	componentconfigs.DefaultKubeProxyConfiguration(obj)
+}

cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer_test.go

@@ -16,7 +16,12 @@ limitations under the License.
 
 package fuzzer
 
-import (
+// TODO: Fuzzing rouudtrip tests are currently disabled in the v1.12 cycle due to the
+// v1alpha2 -> v1alpha3 migration. As the ComponentConfigs were embedded in the structs
+// earlier now have moved out it's not possible to do a lossless roundtrip "the normal way"
+// When we support v1alpha3 and higher only, we can reenable this
+
+/*import (
 	"testing"
 
 	"k8s.io/apimachinery/pkg/api/apitesting/roundtrip"
@@ -25,4 +30,4 @@ import (
 
 func TestRoundTripTypes(t *testing.T) {
 	roundtrip.RoundTripTestForAPIGroup(t, scheme.AddToScheme, Funcs)
-}
+}*/

cmd/kubeadm/app/apis/kubeadm/register.go

@@ -47,6 +47,7 @@ func Resource(resource string) schema.GroupResource {
 func addKnownTypes(scheme *runtime.Scheme) error {
 	scheme.AddKnownTypes(SchemeGroupVersion,
 		&InitConfiguration{},
+		&ClusterConfiguration{},
 		&JoinConfiguration{},
 	)
 	return nil

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -19,38 +19,46 @@ package kubeadm
 import (
 	fuzz "github.com/google/gofuzz"
 
-	v1 "k8s.io/api/core/v1"
+	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
 	"k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig"
 )
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
-// InitConfiguration contains a list of elements which make up master's
-// configuration object.
+// InitConfiguration contains a list of fields that are specifically "kubeadm init"-only runtime
+// information. The cluster-wide config is stored in ClusterConfiguration. The InitConfiguration
+// object IS NOT uploaded to the kubeadm-config ConfigMap in the cluster, only the
+// ClusterConfiguration is.
 type InitConfiguration struct {
 	metav1.TypeMeta
 
-	// `kubeadm init`-only information. These fields are solely used the first time `kubeadm init` runs.
-	// After that, the information in the fields ARE NOT uploaded to the `kubeadm-config` ConfigMap
-	// that is used by `kubeadm upgrade` for instance.
+	// ClusterConfiguration holds the cluster-wide information, and embeds that struct (which can be (un)marshalled separately as well)
+	// When InitConfiguration is marshalled to bytes in the external version, this information IS NOT preserved (which can be seen from
+	// the `json:"-"` tag in the external variant of these API types. Here, in the internal version `json:",inline"` is used, which means
+	// that all of ClusterConfiguration's fields will appear as they would be InitConfiguration's fields. This is used in practice solely
+	// in kubeadm API roundtrip unit testing. Check out `cmd/kubeadm/app/util/config/*_test.go` for more information. Normally, the internal
+	// type is NEVER marshalled, but always converted to some external version first.
+	ClusterConfiguration `json:",inline"`
 
 	// BootstrapTokens is respected at `kubeadm init` time and describes a set of Bootstrap Tokens to create.
-	// This information IS NOT uploaded to the kubeadm cluster configmap, partly because of its sensitive nature
 	BootstrapTokens []BootstrapToken
 
 	// NodeRegistration holds fields that relate to registering the new master node to the cluster
 	NodeRegistration NodeRegistrationOptions
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// ClusterConfiguration contains cluster-wide configuration for a kubeadm cluster
+type ClusterConfiguration struct {
+	metav1.TypeMeta
 
 	// ComponentConfigs holds internal ComponentConfig struct types known to kubeadm, should long-term only exist in the internal kubeadm API
 	// +k8s:conversion-gen=false
 	ComponentConfigs ComponentConfigs
 
-	// Cluster-wide configuration
-	// TODO: Move these fields under some kind of ClusterConfiguration or similar struct that describes
-	// one cluster. Eventually we want this kind of spec to align well with the Cluster API spec.
-
 	// API holds configuration for the k8s apiserver.
 	API API
 	// Etcd holds configuration for etcd.
@@ -254,7 +262,6 @@ type ExternalEtcd struct {
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
 // JoinConfiguration contains elements describing a particular node.
-// TODO: This struct should be replaced by dynamic kubelet configuration.
 type JoinConfiguration struct {
 	metav1.TypeMeta
 

cmd/kubeadm/app/apis/kubeadm/v1alpha2/conversion.go

@@ -17,6 +17,8 @@ limitations under the License.
 package v1alpha2
 
 import (
+	"unsafe"
+
 	"k8s.io/apimachinery/pkg/conversion"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	"k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs"
@@ -50,6 +52,31 @@ func Convert_v1alpha2_InitConfiguration_To_kubeadm_InitConfiguration(in *InitCon
 		}
 	}
 
+	if err := Convert_v1alpha2_API_To_kubeadm_API(&in.API, &out.ClusterConfiguration.API, s); err != nil {
+		return err
+	}
+	if err := Convert_v1alpha2_Etcd_To_kubeadm_Etcd(&in.Etcd, &out.ClusterConfiguration.Etcd, s); err != nil {
+		return err
+	}
+	if err := Convert_v1alpha2_Networking_To_kubeadm_Networking(&in.Networking, &out.ClusterConfiguration.Networking, s); err != nil {
+		return err
+	}
+	out.ClusterConfiguration.KubernetesVersion = in.KubernetesVersion
+	out.ClusterConfiguration.APIServerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.APIServerExtraArgs))
+	out.ClusterConfiguration.ControllerManagerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ControllerManagerExtraArgs))
+	out.ClusterConfiguration.SchedulerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.SchedulerExtraArgs))
+	out.ClusterConfiguration.APIServerExtraVolumes = *(*[]kubeadm.HostPathMount)(unsafe.Pointer(&in.APIServerExtraVolumes))
+	out.ClusterConfiguration.ControllerManagerExtraVolumes = *(*[]kubeadm.HostPathMount)(unsafe.Pointer(&in.ControllerManagerExtraVolumes))
+	out.ClusterConfiguration.SchedulerExtraVolumes = *(*[]kubeadm.HostPathMount)(unsafe.Pointer(&in.SchedulerExtraVolumes))
+	out.ClusterConfiguration.APIServerCertSANs = *(*[]string)(unsafe.Pointer(&in.APIServerCertSANs))
+	out.ClusterConfiguration.CertificatesDir = in.CertificatesDir
+	out.ClusterConfiguration.ImageRepository = in.ImageRepository
+	out.ClusterConfiguration.UnifiedControlPlaneImage = in.UnifiedControlPlaneImage
+	if err := Convert_v1alpha2_AuditPolicyConfiguration_To_kubeadm_AuditPolicyConfiguration(&in.AuditPolicyConfiguration, &out.ClusterConfiguration.AuditPolicyConfiguration, s); err != nil {
+		return err
+	}
+	out.ClusterConfiguration.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.FeatureGates))
+	out.ClusterConfiguration.ClusterName = in.ClusterName
 	return nil
 }
 
@@ -76,5 +103,32 @@ func Convert_kubeadm_InitConfiguration_To_v1alpha2_InitConfiguration(in *kubeadm
 			return err
 		}
 	}
+
+	if err := Convert_kubeadm_API_To_v1alpha2_API(&in.ClusterConfiguration.API, &out.API, s); err != nil {
+		return err
+	}
+	if err := Convert_kubeadm_Etcd_To_v1alpha2_Etcd(&in.ClusterConfiguration.Etcd, &out.Etcd, s); err != nil {
+		return err
+	}
+	if err := Convert_kubeadm_Networking_To_v1alpha2_Networking(&in.ClusterConfiguration.Networking, &out.Networking, s); err != nil {
+		return err
+	}
+	out.KubernetesVersion = in.ClusterConfiguration.KubernetesVersion
+	out.APIServerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ClusterConfiguration.APIServerExtraArgs))
+	out.ControllerManagerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ClusterConfiguration.ControllerManagerExtraArgs))
+	out.SchedulerExtraArgs = *(*map[string]string)(unsafe.Pointer(&in.ClusterConfiguration.SchedulerExtraArgs))
+	out.APIServerExtraVolumes = *(*[]HostPathMount)(unsafe.Pointer(&in.ClusterConfiguration.APIServerExtraVolumes))
+	out.ControllerManagerExtraVolumes = *(*[]HostPathMount)(unsafe.Pointer(&in.ClusterConfiguration.ControllerManagerExtraVolumes))
+	out.SchedulerExtraVolumes = *(*[]HostPathMount)(unsafe.Pointer(&in.ClusterConfiguration.SchedulerExtraVolumes))
+	out.APIServerCertSANs = *(*[]string)(unsafe.Pointer(&in.ClusterConfiguration.APIServerCertSANs))
+	out.CertificatesDir = in.ClusterConfiguration.CertificatesDir
+	out.ImageRepository = in.ClusterConfiguration.ImageRepository
+	out.UnifiedControlPlaneImage = in.ClusterConfiguration.UnifiedControlPlaneImage
+	if err := Convert_kubeadm_AuditPolicyConfiguration_To_v1alpha2_AuditPolicyConfiguration(&in.ClusterConfiguration.AuditPolicyConfiguration, &out.AuditPolicyConfiguration, s); err != nil {
+		return err
+	}
+	out.FeatureGates = *(*map[string]bool)(unsafe.Pointer(&in.ClusterConfiguration.FeatureGates))
+	out.ClusterName = in.ClusterConfiguration.ClusterName
+
 	return nil
 }