Skip to content

Commit

Permalink
feat: custom image settings for k8s upgrade
Browse files Browse the repository at this point in the history
Allows to use custom registry/images.

Fixes: #8275

Co-authored-by:  @g3offrey
Signed-off-by: Matthieu STROHL <mstrohl@dive-in-it.com>
Signed-off-by: Andrey Smirnov <andrey.smirnov@siderolabs.com>
  • Loading branch information
mstrohl authored and smira committed Feb 15, 2024
1 parent fa3b933 commit 3fe82ec
Show file tree
Hide file tree
Showing 8 changed files with 86 additions and 47 deletions.
7 changes: 7 additions & 0 deletions cmd/talosctl/cmd/talos/upgrade-k8s.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,13 @@ func init() {
upgradeK8sCmd.Flags().BoolVar(&upgradeOptions.DryRun, "dry-run", false, "skip the actual upgrade and show the upgrade plan instead")
upgradeK8sCmd.Flags().BoolVar(&upgradeOptions.PrePullImages, "pre-pull-images", true, "pre-pull images before upgrade")
upgradeK8sCmd.Flags().BoolVar(&upgradeOptions.UpgradeKubelet, "upgrade-kubelet", true, "upgrade kubelet service")

upgradeK8sCmd.Flags().StringVar(&upgradeOptions.KubeletImage, "kubelet-image", constants.KubeletImage, "kubelet image to use")
upgradeK8sCmd.Flags().StringVar(&upgradeOptions.APIServerImage, "apiserver-image", constants.KubernetesAPIServerImage, "kube-apiserver image to use")
upgradeK8sCmd.Flags().StringVar(&upgradeOptions.ControllerManagerImage, "controller-manager-image", constants.KubernetesControllerManagerImage, "kube-controller-manager image to use")
upgradeK8sCmd.Flags().StringVar(&upgradeOptions.SchedulerImage, "scheduler-image", constants.KubernetesSchedulerImage, "kube-scheduler image to use")
upgradeK8sCmd.Flags().StringVar(&upgradeOptions.ProxyImage, "proxy-image", constants.KubeProxyImage, "kube-proxy image to use")

addCommand(upgradeK8sCmd)
}

Expand Down
7 changes: 7 additions & 0 deletions hack/release.toml
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,13 @@ config:
- content: MONITOR ${upsmonHost} 1 remote pass password
mountPath: /usr/local/etc/nut/upsmon.conf
```
"""

[notes.k8supgrade]
title = "Kubernetes Upgrade"
description = """\
The command `talosctl upgrade-k8s` now supports specifying custom image references for Kubernetes components via `--*-image` flags.
The default behavior is unchanged, and the flags are optional.
"""

[make_deps]
Expand Down
6 changes: 6 additions & 0 deletions internal/integration/provision/provision.go
Original file line number Diff line number Diff line change
Expand Up @@ -417,6 +417,12 @@ func (suite *BaseSuite) upgradeKubernetes(fromVersion, toVersion string, skipKub
UpgradeKubelet: !skipKubeletUpgrade,
PrePullImages: true,

KubeletImage: constants.KubeletImage,
APIServerImage: constants.KubernetesAPIServerImage,
ControllerManagerImage: constants.KubernetesControllerManagerImage,
SchedulerImage: constants.KubernetesSchedulerImage,
ProxyImage: constants.KubeProxyImage,

EncoderOpt: encoder.WithComments(encoder.CommentsAll),
}

Expand Down
3 changes: 1 addition & 2 deletions pkg/cluster/kubernetes/kubelet.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import (
"github.com/siderolabs/talos/pkg/kubernetes"
"github.com/siderolabs/talos/pkg/machinery/client"
v1alpha1config "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1"
"github.com/siderolabs/talos/pkg/machinery/constants"
"github.com/siderolabs/talos/pkg/machinery/resources/k8s"
"github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1"
)
Expand Down Expand Up @@ -199,7 +198,7 @@ func upgradeKubeletPatcher(
}
}

image := fmt.Sprintf("%s:v%s", constants.KubeletImage, options.Path.ToVersion())
image := fmt.Sprintf("%s:v%s", options.KubeletImage, options.Path.ToVersion())

if oldImage == image {
return errUpdateSkipped
Expand Down
79 changes: 43 additions & 36 deletions pkg/cluster/kubernetes/talos_managed.go
Original file line number Diff line number Diff line change
Expand Up @@ -124,9 +124,9 @@ func Upgrade(ctx context.Context, cluster UpgradeProvider, options UpgradeOption

func prePullImages(ctx context.Context, talosClient *client.Client, options UpgradeOptions) error {
for _, imageRef := range []string{
fmt.Sprintf("%s:v%s", constants.KubernetesAPIServerImage, options.Path.ToVersion()),
fmt.Sprintf("%s:v%s", constants.KubernetesControllerManagerImage, options.Path.ToVersion()),
fmt.Sprintf("%s:v%s", constants.KubernetesSchedulerImage, options.Path.ToVersion()),
fmt.Sprintf("%s:v%s", options.APIServerImage, options.Path.ToVersion()),
fmt.Sprintf("%s:v%s", options.ControllerManagerImage, options.Path.ToVersion()),
fmt.Sprintf("%s:v%s", options.SchedulerImage, options.Path.ToVersion()),
} {
for _, node := range options.controlPlaneNodes {
options.Log(" > %q: pre-pulling %s", node, imageRef)
Expand All @@ -146,7 +146,7 @@ func prePullImages(ctx context.Context, talosClient *client.Client, options Upgr
return nil
}

imageRef := fmt.Sprintf("%s:v%s", constants.KubeletImage, options.Path.ToVersion())
imageRef := fmt.Sprintf("%s:v%s", options.KubeletImage, options.Path.ToVersion())

for _, node := range append(append([]string(nil), options.controlPlaneNodes...), options.workerNodes...) {
options.Log(" > %q: pre-pulling %s", node, imageRef)
Expand Down Expand Up @@ -206,7 +206,7 @@ func patchKubeProxy(options UpgradeOptions) func(config *v1alpha1config.Config)
config.ClusterConfig.ProxyConfig = &v1alpha1config.ProxyConfig{}
}

config.ClusterConfig.ProxyConfig.ContainerImage = fmt.Sprintf("%s:v%s", constants.KubeProxyImage, options.Path.ToVersion())
config.ClusterConfig.ProxyConfig.ContainerImage = fmt.Sprintf("%s:v%s", options.ProxyImage, options.Path.ToVersion())

return nil
}
Expand Down Expand Up @@ -304,7 +304,23 @@ func upgradeStaticPodOnNode(ctx context.Context, cluster UpgradeProvider, option

var errUpdateSkipped = errors.New("update skipped")

//nolint:gocyclo,cyclop
func staticPodImage(logUpdate func(oldImage string), imageName, containerImage, configImage string, options UpgradeOptions) (string, error) {
image := fmt.Sprintf("%s:v%s", imageName, options.Path.ToVersion())

if containerImage == image || configImage == image {
return "", errUpdateSkipped
}

logUpdate(containerImage)

if options.DryRun {
return "", errUpdateSkipped
}

return image, nil
}

//nolint:gocyclo
func upgradeStaticPodPatcher(options UpgradeOptions, service string, configResource resource.Resource) func(config *v1alpha1config.Config) error {
return func(config *v1alpha1config.Config) error {
if config.ClusterConfig == nil {
Expand Down Expand Up @@ -349,16 +365,13 @@ func upgradeStaticPodPatcher(options UpgradeOptions, service string, configResou
config.ClusterConfig.APIServerConfig = &v1alpha1config.APIServerConfig{}
}

image := fmt.Sprintf("%s:v%s", constants.KubernetesAPIServerImage, options.Path.ToVersion())

if config.ClusterConfig.APIServerConfig.ContainerImage == image || configImage == image {
return errUpdateSkipped
}

logUpdate(config.ClusterConfig.APIServerConfig.ContainerImage)

if options.DryRun {
return errUpdateSkipped
image, err := staticPodImage(logUpdate,
options.APIServerImage,
config.ClusterConfig.APIServerConfig.ContainerImage,
configImage,
options)
if err != nil {
return err
}

config.ClusterConfig.APIServerConfig.ContainerImage = image
Expand All @@ -367,16 +380,13 @@ func upgradeStaticPodPatcher(options UpgradeOptions, service string, configResou
config.ClusterConfig.ControllerManagerConfig = &v1alpha1config.ControllerManagerConfig{}
}

image := fmt.Sprintf("%s:v%s", constants.KubernetesControllerManagerImage, options.Path.ToVersion())

if config.ClusterConfig.ControllerManagerConfig.ContainerImage == image || configImage == image {
return errUpdateSkipped
}

logUpdate(config.ClusterConfig.ControllerManagerConfig.ContainerImage)

if options.DryRun {
return errUpdateSkipped
image, err := staticPodImage(logUpdate,
options.ControllerManagerImage,
config.ClusterConfig.ControllerManagerConfig.ContainerImage,
configImage,
options)
if err != nil {
return err
}

config.ClusterConfig.ControllerManagerConfig.ContainerImage = image
Expand All @@ -385,16 +395,13 @@ func upgradeStaticPodPatcher(options UpgradeOptions, service string, configResou
config.ClusterConfig.SchedulerConfig = &v1alpha1config.SchedulerConfig{}
}

image := fmt.Sprintf("%s:v%s", constants.KubernetesSchedulerImage, options.Path.ToVersion())

if config.ClusterConfig.SchedulerConfig.ContainerImage == image || configImage == image {
return errUpdateSkipped
}

logUpdate(config.ClusterConfig.SchedulerConfig.ContainerImage)

if options.DryRun {
return errUpdateSkipped
image, err := staticPodImage(logUpdate,
options.SchedulerImage,
config.ClusterConfig.SchedulerConfig.ContainerImage,
configImage,
options)
if err != nil {
return err
}

config.ClusterConfig.SchedulerConfig.ContainerImage = image
Expand Down
6 changes: 6 additions & 0 deletions pkg/cluster/kubernetes/upgrade.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,12 @@ type UpgradeOptions struct {
DryRun bool
EncoderOpt encoder.Option

KubeletImage string
APIServerImage string
ControllerManagerImage string
SchedulerImage string
ProxyImage string

controlPlaneNodes []string
workerNodes []string
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,8 @@ This command runs in several phases:
If the command fails for any reason, it can be safely restarted to continue the upgrade process from the moment of the failure.
> Note: When using custom/overridden Kubernetes component images, use flags `--*-image` to override the default image names.
## Manual Kubernetes Upgrade
Kubernetes can be upgraded manually by following the steps outlined below.
Expand Down
23 changes: 14 additions & 9 deletions website/content/v1.7/reference/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -2887,15 +2887,20 @@ talosctl upgrade-k8s [flags]
### Options

```
--dry-run skip the actual upgrade and show the upgrade plan instead
--endpoint string the cluster control plane endpoint
--from string the Kubernetes control plane version to upgrade from
-h, --help help for upgrade-k8s
--pre-pull-images pre-pull images before upgrade (default true)
--to string the Kubernetes control plane version to upgrade to (default "1.29.1")
--upgrade-kubelet upgrade kubelet service (default true)
--with-docs patch all machine configs adding the documentation for each field (default true)
--with-examples patch all machine configs with the commented examples (default true)
--apiserver-image string kube-apiserver image to use (default "registry.k8s.io/kube-apiserver")
--controller-manager-image string kube-controller-manager image to use (default "registry.k8s.io/kube-controller-manager")
--dry-run skip the actual upgrade and show the upgrade plan instead
--endpoint string the cluster control plane endpoint
--from string the Kubernetes control plane version to upgrade from
-h, --help help for upgrade-k8s
--kubelet-image string kubelet image to use (default "ghcr.io/siderolabs/kubelet")
--pre-pull-images pre-pull images before upgrade (default true)
--proxy-image string kube-proxy image to use (default "registry.k8s.io/kube-proxy")
--scheduler-image string kube-scheduler image to use (default "registry.k8s.io/kube-scheduler")
--to string the Kubernetes control plane version to upgrade to (default "1.29.1")
--upgrade-kubelet upgrade kubelet service (default true)
--with-docs patch all machine configs adding the documentation for each field (default true)
--with-examples patch all machine configs with the commented examples (default true)
```

### Options inherited from parent commands
Expand Down

0 comments on commit 3fe82ec

Please sign in to comment.