Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

kubeadm: Use separate YAML docs for the kubelet and kube-proxy ComponentConfigs #65787

Make kubeadm support {un,}marshalling ComponentConfig structs as diff…

…erent YAML documents
  • Loading branch information...
luxas committed Jul 8, 2018
commit 495ac9883d8b92906b26f1fe27e97ef167790cb9
@@ -89,6 +89,7 @@ func NewCmdConfig(out io.Writer) *cobra.Command {
}

// NewCmdConfigPrintDefault returns cobra.Command for "kubeadm config print-default" command
// TODO: Make it possible to print the defaults for the componentconfig API objects, and default to printing them out as well
func NewCmdConfigPrintDefault(out io.Writer) *cobra.Command {
apiObjects := []string{}
cmd := &cobra.Command{
@@ -143,7 +144,7 @@ func getDefaultAPIObjectBytes(apiObject string) ([]byte, error) {
if err != nil {
return []byte{}, err
}
return kubeadmutil.MarshalToYamlForCodecs(internalcfg, kubeadmapiv1alpha3.SchemeGroupVersion, kubeadmscheme.Codecs)
return configutil.MarshalKubeadmConfigObject(internalcfg)
}

// NewCmdConfigMigrate returns cobra.Command for "kubeadm config migrate" command
@@ -175,7 +176,7 @@ func NewCmdConfigMigrate(out io.Writer) *cobra.Command {
internalcfg, err := configutil.AnyConfigFileAndDefaultsToInternal(oldCfgPath)
kubeadmutil.CheckErr(err)

outputBytes, err := kubeadmutil.MarshalToYamlForCodecs(internalcfg, kubeadmapiv1alpha3.SchemeGroupVersion, kubeadmscheme.Codecs)
outputBytes, err := configutil.MarshalKubeadmConfigObject(internalcfg)
kubeadmutil.CheckErr(err)

if newCfgPath == "" {
@@ -30,13 +30,10 @@ import (
fakediscovery "k8s.io/client-go/discovery/fake"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
@@ -118,10 +115,7 @@ func printConfiguration(cfg *kubeadmapi.MasterConfiguration, w io.Writer) {
return
}

externalcfg := &kubeadmapiv1alpha3.MasterConfiguration{}
kubeadmscheme.Scheme.Convert(cfg, externalcfg, nil)

cfgYaml, err := kubeadmutil.MarshalToYamlForCodecs(externalcfg, kubeadmapiv1alpha3.SchemeGroupVersion, kubeadmscheme.Codecs)
cfgYaml, err := configutil.MarshalKubeadmConfigObject(cfg)
if err == nil {
fmt.Fprintln(w, "[upgrade/config] Configuration used:")

@@ -29,11 +29,10 @@ import (
clientset "k8s.io/client-go/kubernetes"
clientsetscheme "k8s.io/client-go/kubernetes/scheme"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
kubeproxyconfigscheme "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/scheme"
kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
)

const (
@@ -57,7 +56,7 @@ func EnsureProxyAddon(cfg *kubeadmapi.MasterConfiguration, client clientset.Inte
return err
}

proxyBytes, err := kubeadmutil.MarshalToYamlForCodecs(cfg.ComponentConfigs.KubeProxy, kubeproxyconfigv1alpha1.SchemeGroupVersion, kubeproxyconfigscheme.Codecs)
proxyBytes, err := componentconfigs.Known[componentconfigs.KubeProxyConfigurationKind].Marshal(cfg.ComponentConfigs.KubeProxy)
if err != nil {
return fmt.Errorf("error when marshaling: %v", err)
}
@@ -28,13 +28,11 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
rbachelper "k8s.io/kubernetes/pkg/apis/rbac/v1"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
kubeletconfigscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
"k8s.io/kubernetes/pkg/util/version"
)

@@ -154,14 +152,9 @@ func configMapRBACName(k8sVersion *version.Version) string {
return fmt.Sprintf("%s%d.%d", kubeadmconstants.KubeletBaseConfigMapRolePrefix, k8sVersion.Major(), k8sVersion.Minor())
}

// getConfigBytes marshals a kubeletconfiguration object to bytes
// getConfigBytes marshals a KubeletConfiguration object to bytes
func getConfigBytes(kubeletConfig *kubeletconfig.KubeletConfiguration) ([]byte, error) {
_, kubeletCodecs, err := kubeletconfigscheme.NewSchemeAndCodecs()
if err != nil {
return []byte{}, err
}

return kubeadmutil.MarshalToYamlForCodecs(kubeletConfig, kubeletconfigv1beta1.SchemeGroupVersion, *kubeletCodecs)
return componentconfigs.Known[componentconfigs.KubeletConfigurationKind].Marshal(kubeletConfig)
}

// writeConfigBytesToDisk writes a byte slice down to disk at the specific location of the kubelet config file
@@ -23,11 +23,9 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
)

// UploadConfiguration saves the MasterConfiguration used for later reference (when upgrading for instance)
@@ -41,9 +39,17 @@ func UploadConfiguration(cfg *kubeadmapi.MasterConfiguration, client clientset.I
cfgToUpload.BootstrapTokens = nil
// Clear the NodeRegistration object.
cfgToUpload.NodeRegistration = kubeadmapi.NodeRegistrationOptions{}
// TODO: Reset the .ComponentConfig struct like this:
// cfgToUpload.ComponentConfigs = kubeadmapi.ComponentConfigs{}
// in order to not upload any other components' config to the kubeadm-config
// ConfigMap. The components store their config in their own ConfigMaps.
// Before this line can be uncommented util/config.loadConfigurationBytes()
// needs to support reading the different components' ConfigMaps first.

// Marshal the object into YAML
cfgYaml, err := kubeadmutil.MarshalToYamlForCodecs(cfgToUpload, kubeadmapiv1alpha3.SchemeGroupVersion, kubeadmscheme.Codecs)
cfgYaml, err := configutil.MarshalKubeadmConfigObject(cfgToUpload)
if err != nil {
fmt.Println("err", err.Error())
return err
}

@@ -59,6 +59,7 @@ func loadConfigurationBytes(client clientset.Interface, w io.Writer, logPrefix,
} else if err != nil {
return []byte{}, fmt.Errorf("an unexpected error happened when trying to get the ConfigMap %q in the %s namespace: %v", constants.MasterConfigurationConfigMap, metav1.NamespaceSystem, err)
}
// TODO: Load the kube-proxy and kubelet ComponentConfig ConfigMaps here as different YAML documents and append to the byte slice

fmt.Fprintf(w, "[%s] FYI: You can look at this config file with 'kubectl -n %s get cm %s -oyaml'\n", logPrefix, metav1.NamespaceSystem, constants.MasterConfigurationConfigMap)
return []byte(configMap.Data[constants.MasterConfigurationConfigMapKey]), nil
@@ -25,6 +25,7 @@ import (

"k8s.io/apimachinery/pkg/runtime"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
@@ -53,6 +54,16 @@ func AnyConfigFileAndDefaultsToInternal(cfgPath string) (runtime.Object, error)
return nil, fmt.Errorf("didn't recognize types with GroupVersionKind: %v", gvks)
}

// MarshalKubeadmConfigObject marshals an Object registered in the kubeadm scheme. If the object is a MasterConfiguration, some extra logic is run
func MarshalKubeadmConfigObject(obj runtime.Object) ([]byte, error) {
switch internalcfg := obj.(type) {
case *kubeadmapi.MasterConfiguration:
return MarshalMasterConfigurationToBytes(internalcfg, kubeadmapiv1alpha3.SchemeGroupVersion)
default:
return kubeadmutil.MarshalToYamlForCodecs(obj, kubeadmapiv1alpha3.SchemeGroupVersion, kubeadmscheme.Codecs)
}
}

// DetectUnsupportedVersion reads YAML bytes, extracts the TypeMeta information and errors out with an user-friendly message if the API spec is too old for this kubeadm version
func DetectUnsupportedVersion(b []byte) error {
gvks, err := kubeadmutil.GroupVersionKindsFromBytes(b)
@@ -17,27 +17,36 @@ limitations under the License.
package config

import (
"bytes"
"fmt"
"io/ioutil"
"net"
"reflect"
"sort"

"github.com/golang/glog"

"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
netutil "k8s.io/apimachinery/pkg/util/net"
bootstraputil "k8s.io/client-go/tools/bootstrap/token/util"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
kubeadmapiv1alpha3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha3"
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
"k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
nodeutil "k8s.io/kubernetes/pkg/util/node"
)

// SetInitDynamicDefaults checks and sets configuration values for the MasterConfiguration object
func SetInitDynamicDefaults(cfg *kubeadmapi.MasterConfiguration) error {

// Default all the embedded ComponentConfig structs
componentconfigs.Known.Default(cfg)

// validate cfg.API.AdvertiseAddress.
addressIP := net.ParseIP(cfg.API.AdvertiseAddress)
if addressIP == nil && cfg.API.AdvertiseAddress != "" {
@@ -115,26 +124,73 @@ func ConfigFileAndDefaultsToInternalConfig(cfgPath string, defaultversionedcfg *
return BytesToInternalConfig(b)
}

// Takes passed flags into account; the defaulting is executed once again enforcing assignement of
// Takes passed flags into account; the defaulting is executed once again enforcing assignment of
// static default values to cfg only for values not provided with flags
kubeadmscheme.Scheme.Default(defaultversionedcfg)
kubeadmscheme.Scheme.Convert(defaultversionedcfg, internalcfg, nil)

return defaultAndValidate(internalcfg)
}

// BytesToInternalConfig converts a byte array to an internal, defaulted and validated configuration object
// BytesToInternalConfig converts a byte slice to an internal, defaulted and validated configuration object.
// The byte slice may contain one or many different YAML documents. These YAML documents are parsed one-by-one
// and well-known ComponentConfig GroupVersionKinds are stored inside of the internal MasterConfiguration struct
func BytesToInternalConfig(b []byte) (*kubeadmapi.MasterConfiguration, error) {
internalcfg := &kubeadmapi.MasterConfiguration{}
decodedObjs := map[componentconfigs.RegistrationKind]runtime.Object{}
masterConfigFound := false

if err := DetectUnsupportedVersion(b); err != nil {
return nil, err
}

if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), b, internalcfg); err != nil {
gvkmap, err := kubeadmutil.SplitYAMLDocuments(b)
if err != nil {
return nil, err
}

for gvk, fileContent := range gvkmap {
// Try to get the registration for the ComponentConfig based on the kind
regKind := componentconfigs.RegistrationKind(gvk.Kind)
registration, found := componentconfigs.Known[regKind]
if found {
// Unmarshal the bytes from the YAML document into a runtime.Object containing the ComponentConfiguration struct
obj, err := registration.Unmarshal(fileContent)
if err != nil {
return nil, err
}
decodedObjs[regKind] = obj
continue
}

if gvk.Kind == kubeadmconstants.MasterConfigurationKind {
if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), fileContent, internalcfg); err != nil {
return nil, err
}
masterConfigFound = true
continue
}

fmt.Printf("[config] WARNING: Ignored YAML document with GroupVersionKind %v\n", gvk)
}
// Just as an extra safety check, don't proceed if a MasterConfiguration object wasn't found
if !masterConfigFound {
return nil, fmt.Errorf("no MasterConfiguration kind was found in the YAML file")
}

// Save the loaded ComponentConfig objects in the internalcfg object
for kind, obj := range decodedObjs {
registration, found := componentconfigs.Known[kind]
if found {
if ok := registration.SetToInternalConfig(obj, internalcfg); !ok {
return nil, fmt.Errorf("couldn't save componentconfig value for kind %q", string(kind))
}
} else {
// This should never happen in practice
fmt.Printf("[config] WARNING: Decoded a kind that couldn't be saved to the internal configuration: %q\n", string(kind))
}
}

return defaultAndValidate(internalcfg)
}

@@ -147,5 +203,80 @@ func defaultAndValidate(cfg *kubeadmapi.MasterConfiguration) (*kubeadmapi.Master
if err := validation.ValidateMasterConfiguration(cfg).ToAggregate(); err != nil {
return nil, err
}

return cfg, nil
}

func defaultedInternalConfig() *kubeadmapi.MasterConfiguration {
externalcfg := &kubeadmapiv1alpha3.MasterConfiguration{}
internalcfg := &kubeadmapi.MasterConfiguration{}

kubeadmscheme.Scheme.Default(externalcfg)
kubeadmscheme.Scheme.Convert(externalcfg, internalcfg, nil)

// Default the embedded ComponentConfig structs
componentconfigs.Known.Default(internalcfg)
return internalcfg
}

// MarshalMasterConfigurationToBytes marshals the internal MasterConfiguration object to bytes. It writes the embedded
// ComponentConfiguration objects out as separate YAML documents
func MarshalMasterConfigurationToBytes(cfg *kubeadmapi.MasterConfiguration, gv schema.GroupVersion) ([]byte, error) {
masterbytes, err := kubeadmutil.MarshalToYamlForCodecs(cfg, gv, kubeadmscheme.Codecs)
if err != nil {
return []byte{}, err
}
allFiles := [][]byte{masterbytes}
componentConfigContent := map[string][]byte{}

// If the specified groupversion is targeting the internal type, don't print the extra componentconfig YAML documents
if gv.Version != runtime.APIVersionInternal {

defaultedcfg := defaultedInternalConfig()

for kind, registration := range componentconfigs.Known {
// If the ComponentConfig struct for the current registration is nil, skip it when marshalling
realobj, ok := registration.GetFromInternalConfig(cfg)
if !ok {
continue
}

defaultedobj, ok := registration.GetFromInternalConfig(defaultedcfg)
// Invalid: The caller asked to not print the componentconfigs if defaulted, but defaultComponentConfigs() wasn't able to create default objects to use for reference
if !ok {
return []byte{}, fmt.Errorf("couldn't create a default componentconfig object")
}

// If the real ComponentConfig object differs from the default, print it out. If not, there's no need to print it out, so skip it
if !reflect.DeepEqual(realobj, defaultedobj) {
contentBytes, err := registration.Marshal(realobj)
if err != nil {
return []byte{}, err
}
componentConfigContent[string(kind)] = contentBytes
}
}
}

// Sort the ComponentConfig files by kind when marshalling
sortedComponentConfigFiles := consistentOrderByteSlice(componentConfigContent)
allFiles = append(allFiles, sortedComponentConfigFiles...)
return bytes.Join(allFiles, []byte(kubeadmconstants.YAMLDocumentSeparator)), nil
}

// consistentOrderByteSlice takes a map of a string key and a byte slice, and returns a byte slice of byte slices
// with consistent ordering, where the keys in the map determine the ordering of the return value. This has to be
// done as the order of a for...range loop over a map in go is undeterministic, and could otherwise lead to flakes
// in e.g. unit tests when marshalling content with a random order
func consistentOrderByteSlice(content map[string][]byte) [][]byte {
keys := []string{}
sortedContent := [][]byte{}
for key := range content {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
sortedContent = append(sortedContent, content[key])
}
return sortedContent
}
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.