Skip to content

Commit

Permalink
Merge pull request kubernetes#67944 from fabriziopandini/kubeadm-conf…
Browse files Browse the repository at this point in the history
…ig-configMap

Automatic merge from submit-queue (batch tested with PRs 63011, 68089, 67944, 68132). If you want to cherry-pick this change to another branch, please follow the instructions here: https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md.

Kubeadm upload and fetch of kubeam config v1alpha3

**What this PR does / why we need it**:
This PR implements upload and fetch of kubeam config v1alpha3 from cluster.

More in detail:
In upload, `kubeadm-config` gets
- `ClusterConfiguration` (without components config which are already stored in separated ConfigMaps)
- `ClusterStatus`(initialised or updated with the API endpoint of the current node)

During fetch `InitConfiguration` is composed with:
- `ClusterConfiguration` from `kubeadm-config`
- The `APIEndpoint` of the current node from `ClusterStatus` in `kubeadm-config`
- Component configs from corresponding ConfigMaps

**Which issue(s) this PR fixes** :
refs kubernetes/kubeadm#911, refs kubernetes/kubeadm#963

**Special notes for your reviewer**:
In order to implement this it was necessary to extend current component config management with a new GetFromConfigMap operation. This is implemented in a separated commit "
implement component configs GetFromConfigMap".
The real change build on this (commi "upload and fetch kubeadm v1alpha3")

**Release note**:
```release-note
NONE
```

/cc @kubernetes/sig-cluster-lifecycle-pr-reviews
/sig cluster-lifecycle
/area kubeadm
/kind enhancement
/assign @luxas
/assign @timothysc
/cc @chuckha @rosti @neolit123 @liztio
  • Loading branch information
Kubernetes Submit Queue committed Sep 4, 2018
2 parents 4743489 + d9b4b1f commit 5540edc
Show file tree
Hide file tree
Showing 24 changed files with 1,360 additions and 242 deletions.
75 changes: 40 additions & 35 deletions cmd/kubeadm/app/cmd/join.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ import (
kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
markmasterphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markmaster"
patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode"
uploadconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig"
"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
Expand Down Expand Up @@ -303,21 +304,22 @@ func (j *Join) Run(out io.Writer) error {
}

// If the node should host a new control plane instance
var initConfiguration *kubeadmapi.InitConfiguration
if j.cfg.ControlPlane == true {
// Retrives the kubeadm configuration used during kubeadm init
glog.V(1).Infoln("[join] retrieving KubeConfig objects")
clusterConfiguration, err := j.FetchInitClusterConfiguration(tlsBootstrapCfg)
initConfiguration, err = j.FetchInitConfiguration(tlsBootstrapCfg)
if err != nil {
return err
}

// injects into the kubeadm configuration used for init the information about the joining node
clusterConfiguration.NodeRegistration = j.cfg.NodeRegistration
clusterConfiguration.APIEndpoint.AdvertiseAddress = j.cfg.APIEndpoint.AdvertiseAddress
// injects into the kubeadm configuration the information about the joining node
initConfiguration.NodeRegistration = j.cfg.NodeRegistration
initConfiguration.APIEndpoint = j.cfg.APIEndpoint

// Checks if the cluster configuration supports
// joining a new control plane instance and if all the necessary certificates are provided
if err = j.CheckIfReadyForAdditionalControlPlane(clusterConfiguration); err != nil {
if err = j.CheckIfReadyForAdditionalControlPlane(initConfiguration); err != nil {
// outputs the not ready for hosting a new control plane instance message
ctx := map[string]string{
"Error": err.Error(),
Expand All @@ -330,11 +332,11 @@ func (j *Join) Run(out io.Writer) error {

// run kubeadm init preflight checks for checking all the prequisites
glog.Infoln("[join] running pre-flight checks before initializing the new control plane instance")
preflight.RunInitMasterChecks(utilsexec.New(), clusterConfiguration, j.ignorePreflightErrors)
preflight.RunInitMasterChecks(utilsexec.New(), initConfiguration, j.ignorePreflightErrors)

// Prepares the node for hosting a new control plane instance by writing necessary
// KubeConfig files, and static pod manifests
if err = j.PrepareForHostingControlPlane(clusterConfiguration); err != nil {
if err = j.PrepareForHostingControlPlane(initConfiguration); err != nil {
return err
}
}
Expand All @@ -351,8 +353,8 @@ func (j *Join) Run(out io.Writer) error {

// if the node is hosting a new control plane instance
if j.cfg.ControlPlane == true {
// Marks the node with master taint and label.
if err := j.MarkMaster(); err != nil {
// Completes the control plane setup
if err := j.PostInstallControlPlane(initConfiguration); err != nil {
return err
}

Expand All @@ -370,81 +372,77 @@ func (j *Join) Run(out io.Writer) error {
return nil
}

// FetchInitClusterConfiguration reads the cluster configuration from the kubeadm-admin configMap,
func (j *Join) FetchInitClusterConfiguration(tlsBootstrapCfg *clientcmdapi.Config) (*kubeadmapi.InitConfiguration, error) {
// FetchInitConfiguration reads the cluster configuration from the kubeadm-admin configMap,
func (j *Join) FetchInitConfiguration(tlsBootstrapCfg *clientcmdapi.Config) (*kubeadmapi.InitConfiguration, error) {
// creates a client to access the cluster using the bootstrap token identity
tlsClient, err := kubeconfigutil.ToClientSet(tlsBootstrapCfg)
if err != nil {
return nil, errors.Wrap(err, "Unable to access the cluster")
}

// Fetches the cluster configuration
kubeadmConfig, err := configutil.FetchConfigFromFileOrCluster(tlsClient, os.Stdout, "join", "")
// Fetches the init configuration
initConfiguration, err := configutil.FetchConfigFromFileOrCluster(tlsClient, os.Stdout, "join", "", true)
if err != nil {
return nil, errors.Wrap(err, "Unable to fetch the kubeadm-config ConfigMap")
}

// Converts public API struct to internal API
clusterConfiguration := &kubeadmapi.InitConfiguration{}
kubeadmscheme.Scheme.Convert(kubeadmConfig, clusterConfiguration, nil)

return clusterConfiguration, nil
return initConfiguration, nil
}

// CheckIfReadyForAdditionalControlPlane ensures that the cluster is in a state that supports
// joining an additional control plane instance and if the node is ready to join
func (j *Join) CheckIfReadyForAdditionalControlPlane(clusterConfiguration *kubeadmapi.InitConfiguration) error {
func (j *Join) CheckIfReadyForAdditionalControlPlane(initConfiguration *kubeadmapi.InitConfiguration) error {
// blocks if the cluster was created without a stable control plane endpoint
if clusterConfiguration.ControlPlaneEndpoint == "" {
if initConfiguration.ControlPlaneEndpoint == "" {
return fmt.Errorf("unable to add a new control plane instance a cluster that doesn't have a stable controlPlaneEndpoint address")
}

// blocks if the cluster was created without an external etcd cluster
if clusterConfiguration.Etcd.External == nil {
if initConfiguration.Etcd.External == nil {
return fmt.Errorf("unable to add a new control plane instance on a cluster that doesn't use an external etcd")
}

// blocks if control plane is self-hosted
if features.Enabled(clusterConfiguration.FeatureGates, features.SelfHosting) {
if features.Enabled(initConfiguration.FeatureGates, features.SelfHosting) {
return fmt.Errorf("self-hosted clusters are deprecated and won't be supported by `kubeadm join --experimental-control-plane`")
}

// blocks if the certificates for the control plane are stored in secrets (instead of the local pki folder)
if features.Enabled(clusterConfiguration.FeatureGates, features.StoreCertsInSecrets) {
if features.Enabled(initConfiguration.FeatureGates, features.StoreCertsInSecrets) {
return fmt.Errorf("certificates stored in secrets, as well as self-hosted clusters are deprecated and won't be supported by `kubeadm join --experimental-control-plane`")
}

// checks if the certificates that must be equal across contolplane instances are provided
if ret, err := certsphase.SharedCertificateExists(clusterConfiguration); !ret {
if ret, err := certsphase.SharedCertificateExists(initConfiguration); !ret {
return err
}

return nil
}

// PrepareForHostingControlPlane makes all preparation activities require for a node hosting a new control plane instance
func (j *Join) PrepareForHostingControlPlane(clusterConfiguration *kubeadmapi.InitConfiguration) error {
func (j *Join) PrepareForHostingControlPlane(initConfiguration *kubeadmapi.InitConfiguration) error {

// Creates the admin kubeconfig file for the admin and for kubeadm itself.
if err := kubeconfigphase.CreateAdminKubeConfigFile(kubeadmconstants.KubernetesDir, clusterConfiguration); err != nil {
if err := kubeconfigphase.CreateAdminKubeConfigFile(kubeadmconstants.KubernetesDir, initConfiguration); err != nil {
return errors.Wrap(err, "error generating the admin kubeconfig file")
}

// Generate missing certificates (if any)
if err := certsphase.CreatePKIAssets(clusterConfiguration); err != nil {
if err := certsphase.CreatePKIAssets(initConfiguration); err != nil {
return err
}

// Generate kubeconfig files for controller manager, scheduler and for the admin/kubeadm itself
// NB. The kubeconfig file for kubelet will be generated by the TLS bootstrap process in
// following steps of the join --experimental-control plane workflow
if err := kubeconfigphase.CreateJoinControlPlaneKubeConfigFiles(kubeadmconstants.KubernetesDir, clusterConfiguration); err != nil {
if err := kubeconfigphase.CreateJoinControlPlaneKubeConfigFiles(kubeadmconstants.KubernetesDir, initConfiguration); err != nil {
return errors.Wrap(err, "error generating kubeconfig files")
}

// Creates static pod manifests file for the control plane components to be deployed on this node
// Static pods will be created and managed by the kubelet as soon as it starts
if err := controlplanephase.CreateInitStaticPodManifestFiles(kubeadmconstants.GetStaticPodDirectory(), clusterConfiguration); err != nil {
if err := controlplanephase.CreateInitStaticPodManifestFiles(kubeadmconstants.GetStaticPodDirectory(), initConfiguration); err != nil {
return errors.Wrap(err, "error creating static pod manifest files for the control plane components")
}

Expand Down Expand Up @@ -491,8 +489,10 @@ func (j *Join) BootstrapKubelet(tlsBootstrapCfg *clientcmdapi.Config) error {
return err
}

// Write env file with flags for the kubelet to use. Also register taints
if err := kubeletphase.WriteKubeletDynamicEnvFile(&j.cfg.NodeRegistration, j.cfg.FeatureGates, true, kubeadmconstants.KubeletRunDirectory); err != nil {
// Write env file with flags for the kubelet to use. We do not need to write the --register-with-taints for the master,
// as we handle that ourselves in the markmaster phase
// TODO: Maybe we want to do that some time in the future, in order to remove some logic from the markmaster phase?
if err := kubeletphase.WriteKubeletDynamicEnvFile(&j.cfg.NodeRegistration, j.cfg.FeatureGates, false, kubeadmconstants.KubeletRunDirectory); err != nil {
return err
}

Expand Down Expand Up @@ -530,17 +530,22 @@ func (j *Join) BootstrapKubelet(tlsBootstrapCfg *clientcmdapi.Config) error {
return nil
}

// MarkMaster marks the new node as master
func (j *Join) MarkMaster() error {
// PostInstallControlPlane marks the new node as master and update the cluster status with information about current node
func (j *Join) PostInstallControlPlane(initConfiguration *kubeadmapi.InitConfiguration) error {
kubeConfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.AdminKubeConfigFileName)

client, err := kubeconfigutil.ClientSetFromFile(kubeConfigFile)
if err != nil {
return errors.Wrap(err, "couldn't create kubernetes client")
}

err = markmasterphase.MarkMaster(client, j.cfg.NodeRegistration.Name, j.cfg.NodeRegistration.Taints)
if err != nil {
glog.V(1).Info("[join] uploading currently used configuration to the cluster")
if err := uploadconfigphase.UploadConfiguration(initConfiguration, client); err != nil {
return fmt.Errorf("error uploading configuration: %v", err)
}

glog.V(1).Info("[join] marking the master with right label")
if err = markmasterphase.MarkMaster(client, initConfiguration.NodeRegistration.Name, initConfiguration.NodeRegistration.Taints); err != nil {
return errors.Wrap(err, "error applying master label and taints")
}

Expand Down
2 changes: 1 addition & 1 deletion cmd/kubeadm/app/cmd/upgrade/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ func enforceRequirements(flags *applyPlanFlags, dryRun bool, newK8sVersion strin

// Fetch the configuration from a file or ConfigMap and validate it
fmt.Println("[upgrade/config] Making sure the configuration is correct:")
cfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "upgrade/config", flags.cfgPath)
cfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "upgrade/config", flags.cfgPath, false)
if err != nil {
if apierrors.IsNotFound(err) {
fmt.Printf("[upgrade/config] In order to upgrade, a ConfigMap called %q in the %s namespace must exist.\n", constants.InitConfigurationConfigMap, metav1.NamespaceSystem)
Expand Down
11 changes: 1 addition & 10 deletions cmd/kubeadm/app/cmd/upgrade/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -149,10 +149,6 @@ func NewCmdUpgradeControlPlane() *cobra.Command {

options.AddKubeConfigFlag(cmd.Flags(), &flags.kubeConfigPath)
cmd.Flags().BoolVar(&flags.dryRun, "dry-run", flags.dryRun, "Do not change any state, just output the actions that would be performed.")

//TODO: following values should retrieved form the kubeadm-config config map; remove as soon as the new config wil be in place
cmd.Flags().StringVar(&flags.advertiseAddress, "apiserver-advertise-address", flags.advertiseAddress, "If the node is joining as a master, the IP address the API Server will advertise it's listening on.")
cmd.Flags().StringVar(&flags.nodeName, "node-name", flags.nodeName, "Specify the node name.")
return cmd
}

Expand Down Expand Up @@ -230,16 +226,11 @@ func RunUpgradeControlPlane(flags *controlplaneUpgradeFlags) error {
waiter := apiclient.NewKubeWaiter(client, upgrade.UpgradeManifestTimeout, os.Stdout)

// Fetches the cluster configuration
cfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "upgrade", "")
cfg, err := configutil.FetchConfigFromFileOrCluster(client, os.Stdout, "upgrade", "", false)
if err != nil {
return fmt.Errorf("Unable to fetch the kubeadm-config ConfigMap: %v", err)
}

//TODO: as soon as the new config wil be in place check if the node is a known control plane instance
// and retrive corresponding infos (now are temporary managed as flag)
cfg.NodeRegistration.Name = flags.nodeName
cfg.APIEndpoint.AdvertiseAddress = flags.advertiseAddress

// Rotate API server certificate if needed
if err := upgrade.BackupAPIServerCertIfNeeded(cfg, flags.dryRun); err != nil {
return fmt.Errorf("Unable to rotate API server certificate: %v", err)
Expand Down
13 changes: 12 additions & 1 deletion cmd/kubeadm/app/componentconfigs/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = [
"config.go",
"defaults.go",
"registrations.go",
"scheme.go",
Expand All @@ -19,12 +20,14 @@ go_library(
"//pkg/kubelet/apis/config/validation:go_default_library",
"//pkg/proxy/apis/config:go_default_library",
"//pkg/proxy/apis/config/validation:go_default_library",
"//pkg/util/version:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/validation/field:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/kube-proxy/config/v1alpha1:go_default_library",
"//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library",
"//vendor/k8s.io/utils/pointer:go_default_library",
Expand All @@ -47,14 +50,22 @@ filegroup(

go_test(
name = "go_default_test",
srcs = ["validation_test.go"],
srcs = [
"config_test.go",
"validation_test.go",
],
embed = [":go_default_library"],
deps = [
"//cmd/kubeadm/app/apis/kubeadm:go_default_library",
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//pkg/kubelet/apis/config:go_default_library",
"//pkg/proxy/apis/config:go_default_library",
"//pkg/util/version:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
"//vendor/k8s.io/utils/pointer:go_default_library",
],
)
80 changes: 80 additions & 0 deletions cmd/kubeadm/app/componentconfigs/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package componentconfigs

import (
"fmt"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
clientset "k8s.io/client-go/kubernetes"
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
kubeproxyconfig "k8s.io/kubernetes/pkg/proxy/apis/config"
"k8s.io/kubernetes/pkg/util/version"
)

// GetFromKubeletConfigMap returns the pointer to the ComponentConfig API object read from the kubelet-config-version
// ConfigMap map stored in the cluster
func GetFromKubeletConfigMap(client clientset.Interface, version *version.Version) (runtime.Object, error) {

// Read the ConfigMap from the cluster based on what version the kubelet is
configMapName := kubeadmconstants.GetKubeletConfigMapName(version)
kubeletCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(configMapName, metav1.GetOptions{})
if err != nil {
return nil, err
}

kubeletConfigData, ok := kubeletCfg.Data[kubeadmconstants.KubeletBaseConfigurationConfigMapKey]
if !ok {
return nil, fmt.Errorf("unexpected error when reading %s ConfigMap: %s key value pair missing", configMapName, kubeadmconstants.KubeletBaseConfigurationConfigMapKey)
}

// Decodes the kubeletConfigData into the internal component config
obj := &kubeletconfig.KubeletConfiguration{}
err = unmarshalObject(obj, []byte(kubeletConfigData))
if err != nil {
return nil, err
}

return obj, nil
}

// GetFromKubeProxyConfigMap returns the pointer to the ComponentConfig API object read from the kube-proxy
// ConfigMap map stored in the cluster
func GetFromKubeProxyConfigMap(client clientset.Interface, version *version.Version) (runtime.Object, error) {

// Read the ConfigMap from the cluster
kubeproxyCfg, err := client.CoreV1().ConfigMaps(metav1.NamespaceSystem).Get(kubeadmconstants.KubeProxyConfigMap, metav1.GetOptions{})
if err != nil {
return nil, err
}

kubeproxyConfigData, ok := kubeproxyCfg.Data[kubeadmconstants.KubeProxyConfigMapKey]
if !ok {
return nil, fmt.Errorf("unexpected error when reading %s ConfigMap: %s key value pair missing", kubeadmconstants.KubeProxyConfigMap, kubeadmconstants.KubeProxyConfigMapKey)
}

// Decodes the Config map dat into the internal component config
obj := &kubeproxyconfig.KubeProxyConfiguration{}
err = unmarshalObject(obj, []byte(kubeproxyConfigData))
if err != nil {
return nil, err
}

return obj, nil
}
Loading

0 comments on commit 5540edc

Please sign in to comment.