Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

kubeadm: delete pre-pull ds and add pull images check in upgrade apply and upgrade node #90788

Merged
merged 2 commits into from May 12, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 4 additions & 0 deletions cmd/kubeadm/app/cmd/phases/upgrade/node/BUILD
Expand Up @@ -6,6 +6,7 @@ go_library(
"controlplane.go",
"data.go",
"kubeletconfig.go",
"preflight.go",
],
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/upgrade/node",
visibility = ["//visibility:public"],
Expand All @@ -17,11 +18,14 @@ go_library(
"//cmd/kubeadm/app/constants:go_default_library",
"//cmd/kubeadm/app/phases/kubelet:go_default_library",
"//cmd/kubeadm/app/phases/upgrade:go_default_library",
"//cmd/kubeadm/app/preflight:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//cmd/kubeadm/app/util/dryrun:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/version:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
],
)

Expand Down
2 changes: 2 additions & 0 deletions cmd/kubeadm/app/cmd/phases/upgrade/node/data.go
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package node

import (
"k8s.io/apimachinery/pkg/util/sets"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
)
Expand All @@ -31,5 +32,6 @@ type Data interface {
Cfg() *kubeadmapi.InitConfiguration
IsControlPlaneNode() bool
Client() clientset.Interface
IgnorePreflightErrors() sets.String
KustomizeDir() string
}
73 changes: 73 additions & 0 deletions cmd/kubeadm/app/cmd/phases/upgrade/node/preflight.go
@@ -0,0 +1,73 @@
/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package node

import (
"fmt"

"github.com/pkg/errors"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
utilsexec "k8s.io/utils/exec"
)

// NewPreflightPhase creates a kubeadm workflow phase that implements preflight checks for a new node join
func NewPreflightPhase() workflow.Phase {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like the fact, that we are introducing a new phase here. Makes things much more symmetric with the rest of the phased commands.

return workflow.Phase{
Name: "preflight",
Short: "Run upgrade node pre-flight checks",
Long: "Run pre-flight checks for kubeadm upgrade node.",
Run: runPreflight,
InheritFlags: []string{
options.IgnorePreflightErrors,
},
}
}

// runPreflight executes preflight checks logic.
func runPreflight(c workflow.RunData) error {
data, ok := c.(Data)
if !ok {
return errors.New("preflight phase invoked with an invalid data struct")
}
fmt.Println("[preflight] Running pre-flight checks")

// First, check if we're root separately from the other preflight checks and fail fast
if err := preflight.RunRootCheckOnly(data.IgnorePreflightErrors()); err != nil {
return err
}

// if this is a control-plane node, pull the basic images
if data.IsControlPlaneNode() {
if !data.DryRun() {
fmt.Println("[preflight] Pulling images required for setting up a Kubernetes cluster")
fmt.Println("[preflight] This might take a minute or two, depending on the speed of your internet connection")
fmt.Println("[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'")
if err := preflight.RunPullImagesCheck(utilsexec.New(), data.Cfg(), data.IgnorePreflightErrors()); err != nil {
return err
}
} else {
fmt.Println("[preflight] Would pull the required images (like 'kubeadm config images pull')")
}
} else {
fmt.Println("[preflight] Skipping prepull. Not a control plane node.")
return nil
}

return nil
}
1 change: 1 addition & 0 deletions cmd/kubeadm/app/cmd/upgrade/BUILD
Expand Up @@ -42,6 +42,7 @@ go_library(
"//vendor/github.com/spf13/cobra:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
],
)

Expand Down
30 changes: 15 additions & 15 deletions cmd/kubeadm/app/cmd/upgrade/apply.go
Expand Up @@ -22,17 +22,19 @@ import (

"github.com/pkg/errors"
"github.com/spf13/cobra"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/version"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/klog"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
"k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/features"
"k8s.io/kubernetes/cmd/kubeadm/app/phases/upgrade"
"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
utilsexec "k8s.io/utils/exec"
)

const (
Expand Down Expand Up @@ -89,6 +91,8 @@ func NewCmdApply(apf *applyPlanFlags) *cobra.Command {
cmd.Flags().BoolVar(&flags.etcdUpgrade, "etcd-upgrade", flags.etcdUpgrade, "Perform the upgrade of etcd.")
cmd.Flags().BoolVar(&flags.renewCerts, options.CertificateRenewal, flags.renewCerts, "Perform the renewal of certificates used by component changed during upgrades.")
cmd.Flags().DurationVar(&flags.imagePullTimeout, "image-pull-timeout", flags.imagePullTimeout, "The maximum amount of time to wait for the control plane pods to be downloaded.")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this should go through deprecation...

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

got it

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am not sure if we should keep this flag operational until we can remove it.
@neolit123 @fabriziopandini WDYT?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

technically the flag should be operational until we remove it, which might be a bit difficult given we are removing the backend that it targets.

// TODO: The flag was deprecated in 1.19; remove the flag following a GA deprecation policy of 12 months or 2 releases (whichever is longer)
cmd.Flags().MarkDeprecated("image-pull-timeout", "This flag is deprecated and will be removed in a future version.")
Copy link
Member

@neolit123 neolit123 May 7, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please add a comment on top of this deprecation line:

// TODO: The flag was deprecated in 1.19; remove the flag following a GA deprecation policy of 12 months or 2 releases (whichever is longer)

xref https://kubernetes.io/docs/reference/using-api/deprecation-policy/#deprecating-a-flag-or-cli

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

added

options.AddKustomizePodsFlag(cmd.Flags(), &flags.kustomizeDir)

return cmd
Expand Down Expand Up @@ -145,22 +149,18 @@ func runApply(flags *applyFlags, userVersion string) error {
}
}

// Set the timeout as flags.imagePullTimeout to ensure that Prepuller truly respects 'image-pull-timeout' flag
waiter := getWaiter(flags.dryRun, client, flags.imagePullTimeout)

// Use a prepuller implementation based on creating DaemonSets
// and block until all DaemonSets are ready; then we know for sure that all control plane images are cached locally
klog.V(1).Infoln("[upgrade/apply] creating prepuller")
prepuller := upgrade.NewDaemonSetPrepuller(client, waiter, &cfg.ClusterConfiguration)
componentsToPrepull := constants.ControlPlaneComponents
if cfg.Etcd.External == nil && flags.etcdUpgrade {
componentsToPrepull = append(componentsToPrepull, constants.Etcd)
}
if err := upgrade.PrepullImagesInParallel(prepuller, flags.imagePullTimeout, componentsToPrepull); err != nil {
return errors.Wrap(err, "[upgrade/prepull] Failed prepulled the images for the control plane components error")
if !flags.dryRun {
fmt.Println("[upgrade/prepull] Pulling images required for setting up a Kubernetes cluster")
fmt.Println("[upgrade/prepull] This might take a minute or two, depending on the speed of your internet connection")
fmt.Println("[upgrade/prepull] You can also perform this action in beforehand using 'kubeadm config images pull'")
if err := preflight.RunPullImagesCheck(utilsexec.New(), cfg, sets.NewString(cfg.NodeRegistration.IgnorePreflightErrors...)); err != nil {
return err
}
} else {
fmt.Println("[upgrade/prepull] Would pull the required images (like 'kubeadm config images pull')")
}

waiter = getWaiter(flags.dryRun, client, upgrade.UpgradeManifestTimeout)
waiter := getWaiter(flags.dryRun, client, upgrade.UpgradeManifestTimeout)

// Now; perform the upgrade procedure
klog.V(1).Infoln("[upgrade/apply] performing upgrade")
Expand Down
63 changes: 41 additions & 22 deletions cmd/kubeadm/app/cmd/upgrade/node.go
Expand Up @@ -23,8 +23,10 @@ import (
"github.com/spf13/cobra"
flag "github.com/spf13/pflag"

"k8s.io/apimachinery/pkg/util/sets"
clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
phases "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/upgrade/node"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
Expand All @@ -36,12 +38,13 @@ import (
// Please note that this structure includes the public kubeadm config API, but only a subset of the options
// supported by this api will be exposed as a flag.
type nodeOptions struct {
kubeConfigPath string
kubeletVersion string
etcdUpgrade bool
renewCerts bool
dryRun bool
kustomizeDir string
kubeConfigPath string
kubeletVersion string
etcdUpgrade bool
renewCerts bool
dryRun bool
kustomizeDir string
ignorePreflightErrors []string
}

// compile-time assert that the local data object satisfies the phases data interface.
Expand All @@ -50,14 +53,15 @@ var _ phases.Data = &nodeData{}
// nodeData defines all the runtime information used when running the kubeadm upgrade node worklow;
// this data is shared across all the phases that are included in the workflow.
type nodeData struct {
etcdUpgrade bool
renewCerts bool
dryRun bool
kubeletVersion string
cfg *kubeadmapi.InitConfiguration
isControlPlaneNode bool
client clientset.Interface
kustomizeDir string
etcdUpgrade bool
renewCerts bool
dryRun bool
kubeletVersion string
cfg *kubeadmapi.InitConfiguration
isControlPlaneNode bool
client clientset.Interface
kustomizeDir string
ignorePreflightErrors sets.String
}

// NewCmdNode returns the cobra command for `kubeadm upgrade node`
Expand All @@ -80,6 +84,7 @@ func NewCmdNode() *cobra.Command {
options.AddKustomizePodsFlag(cmd.Flags(), &nodeOptions.kustomizeDir)

// initialize the workflow runner with the list of phases
nodeRunner.AppendPhase(phases.NewPreflightPhase())
nodeRunner.AppendPhase(phases.NewControlPlane())
nodeRunner.AppendPhase(phases.NewKubeletConfigPhase())

Expand Down Expand Up @@ -113,6 +118,7 @@ func addUpgradeNodeFlags(flagSet *flag.FlagSet, nodeOptions *nodeOptions) {
flagSet.MarkDeprecated(options.KubeletVersion, "This flag is deprecated and will be removed in a future version.")
flagSet.BoolVar(&nodeOptions.renewCerts, options.CertificateRenewal, nodeOptions.renewCerts, "Perform the renewal of certificates used by component changed during upgrades.")
flagSet.BoolVar(&nodeOptions.etcdUpgrade, options.EtcdUpgrade, nodeOptions.etcdUpgrade, "Perform the upgrade of etcd.")
flagSet.StringSliceVar(&nodeOptions.ignorePreflightErrors, options.IgnorePreflightErrors, nodeOptions.ignorePreflightErrors, "A list of checks whose errors will be shown as warnings. Example: 'IsPrivilegedUser,Swap'. Value 'all' ignores errors from all checks.")
}

// newNodeData returns a new nodeData struct to be used for the execution of the kubeadm upgrade node workflow.
Expand Down Expand Up @@ -140,15 +146,23 @@ func newNodeData(cmd *cobra.Command, args []string, options *nodeOptions) (*node
return nil, errors.Wrap(err, "unable to fetch the kubeadm-config ConfigMap")
}

ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(options.ignorePreflightErrors, cfg.NodeRegistration.IgnorePreflightErrors)
if err != nil {
return nil, err
}
// Also set the union of pre-flight errors to JoinConfiguration, to provide a consistent view of the runtime configuration:
cfg.NodeRegistration.IgnorePreflightErrors = ignorePreflightErrorsSet.List()

return &nodeData{
etcdUpgrade: options.etcdUpgrade,
renewCerts: options.renewCerts,
dryRun: options.dryRun,
kubeletVersion: options.kubeletVersion,
cfg: cfg,
client: client,
isControlPlaneNode: isControlPlaneNode,
kustomizeDir: options.kustomizeDir,
etcdUpgrade: options.etcdUpgrade,
renewCerts: options.renewCerts,
dryRun: options.dryRun,
kubeletVersion: options.kubeletVersion,
cfg: cfg,
client: client,
isControlPlaneNode: isControlPlaneNode,
kustomizeDir: options.kustomizeDir,
ignorePreflightErrors: ignorePreflightErrorsSet,
}, nil
}

Expand Down Expand Up @@ -191,3 +205,8 @@ func (d *nodeData) Client() clientset.Interface {
func (d *nodeData) KustomizeDir() string {
return d.kustomizeDir
}

// IgnorePreflightErrors returns the list of preflight errors to ignore.
func (d *nodeData) IgnorePreflightErrors() sets.String {
return d.ignorePreflightErrors
}
2 changes: 0 additions & 2 deletions cmd/kubeadm/app/constants/constants.go
Expand Up @@ -187,8 +187,6 @@ const (
TLSBootstrapRetryInterval = 5 * time.Second
// PullImageRetry specifies how many times ContainerRuntime retries when pulling image failed
PullImageRetry = 5
// PrepullImagesInParallelTimeout specifies how long kubeadm should wait for prepulling images in parallel before timing out
PrepullImagesInParallelTimeout = 10 * time.Second

// DefaultControlPlaneTimeout specifies the default control plane (actually API Server) timeout for use by kubeadm
DefaultControlPlaneTimeout = 4 * time.Minute
Expand Down
2 changes: 0 additions & 2 deletions cmd/kubeadm/app/phases/upgrade/BUILD
Expand Up @@ -8,7 +8,6 @@ go_library(
"policy.go",
"postupgrade.go",
"preflight.go",
"prepull.go",
"staticpods.go",
"versiongetter.go",
],
Expand Down Expand Up @@ -76,7 +75,6 @@ go_test(
"compute_test.go",
"policy_test.go",
"postupgrade_test.go",
"prepull_test.go",
"staticpods_test.go",
],
embed = [":go_default_library"],
Expand Down