Update in-cluster images during cluster upgrade

pkg/controller/ground.go

@@ -36,8 +36,10 @@ import (
 	"github.com/sapcc/kubernikus/pkg/util"
 	etcd_util "github.com/sapcc/kubernikus/pkg/util/etcd"
 	helm_util "github.com/sapcc/kubernikus/pkg/util/helm"
+	"github.com/sapcc/kubernikus/pkg/util/k8s"
 	"github.com/sapcc/kubernikus/pkg/util/version"
 	waitutil "github.com/sapcc/kubernikus/pkg/util/wait"
+	appVersion "github.com/sapcc/kubernikus/pkg/version"
 )
 
 const (
@@ -673,6 +675,19 @@ func (op *GroundControl) upgradeKluster(kluster *v1.Kluster, toVersion string) e
 			return errors.Wrap(err, "seed CSI storage classes on upgrade")
 		}
 	}
+	if ok, _ := util.KlusterVersionConstraint(kluster, ">= 1.21"); ok && !kluster.Spec.NoCloud {
+		kubernetes, err := op.Clients.Satellites.ClientFor(kluster)
+		if err != nil {
+			return errors.Wrap(err, "client")
+		}
+		if err := k8s.UpdateDaemonsetImages(kubernetes, "csi-cinder-nodeplugin", "kube-system",
+			"node-driver-registrar", op.Images.Versions[kluster.Spec.Version].CSINodeDriver.String(),
+			"liveness-probe", op.Images.Versions[kluster.Spec.Version].CSILivenessProbe.String(),
+			"cinder-csi-plugin", op.Images.Versions[kluster.Spec.Version].CinderCSIPlugin.String(),
+		); err != nil {
+			return errors.Wrap(err, "Failed to update csi-cinder-nodeplugin daemonset image")
+		}
+	}
 
 	if !kluster.Spec.NoCloud && strings.HasPrefix(toVersion, "1.21") && strings.HasPrefix(kluster.Status.ApiserverVersion, "1.20") {
 		kubernetes, err := op.Clients.Satellites.ClientFor(kluster)
@@ -707,6 +722,21 @@ func (op *GroundControl) upgradeKluster(kluster *v1.Kluster, toVersion string) e
 		}
 	}
 
+	if ok, _ := util.KlusterVersionConstraint(kluster, ">= 1.24"); ok && !kluster.Spec.NoCloud {
+		kubernetes, err := op.Clients.Satellites.ClientFor(kluster)
+		if err != nil {
+			return errors.Wrap(err, "client")
+		}
+		// Make sure we upgrade the kube-proxy image in case we upgrade from 1.24
+		if err := k8s.UpdateDaemonsetImages(kubernetes, "kube-proxy", "kube-system", "proxy", op.Images.Versions[kluster.Spec.Version].KubeProxy.String()); err != nil {
+			return errors.Wrap(err, "Failed to update kube-proxy daemonset image")
+		}
+		wormholeImage := op.Images.Versions[kluster.Spec.Version].Wormhole.Repository + ":" + appVersion.GitCommit
+		if err := k8s.UpdateDaemonsetImages(kubernetes, "wormhole", "kube-system", "wormhole", wormholeImage); err != nil {
+			return errors.Wrap(err, "Failed to update wormhole daemonset image")
+		}
+	}
+
 	if !kluster.Spec.NoCloud && strings.HasPrefix(toVersion, "1.25") && strings.HasPrefix(kluster.Status.ApiserverVersion, "1.24") {
 		kubernetes, err := op.Clients.Satellites.ClientFor(kluster)
 		if err != nil {

pkg/util/k8s/daemonset.go

@@ -0,0 +1,41 @@
+package k8s
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/util/retry"
+)
+
+func UpdateDaemonsetImages(client kubernetes.Interface, name, namespace string, containerAndImage ...string) error {
+	if l := len(containerAndImage); l < 2 || l%2 == 1 {
+		return errors.New("containerAndImage needs to be even number of arguments")
+	}
+	retryErr := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
+		ds, err := client.AppsV1().DaemonSets(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+		if err != nil {
+			return fmt.Errorf("Failed to find daemonset %s/%s: %w", namespace, name, err)
+		}
+		updated := false
+		for i := 0; i < len(containerAndImage)-1; i += 2 {
+			container := containerAndImage[i]
+			newimage := containerAndImage[i+1]
+
+			for i, c := range ds.Spec.Template.Spec.Containers {
+				if c.Name == container && c.Image != newimage {
+					ds.Spec.Template.Spec.Containers[i].Image = newimage
+					updated = true
+				}
+			}
+		}
+		if updated {
+			_, err := client.AppsV1().DaemonSets(namespace).Update(context.TODO(), ds, metav1.UpdateOptions{})
+			return err
+		}
+		return nil
+	})
+	return retryErr
+}