[release/v2.24] Don't propagate cloud-config secret to user cluster

addons/csi/azure-disk/csi-azuredisk-controller.yaml

@@ -18,7 +18,7 @@
 # - image source includes registry templating
 # - removal of tolerations
 # - set spec.replicas=1 since leader elections are configured
-# - mount cloud-config instead of using host path
+# - mount cloud-config-csi instead of using host path
 # - add "cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: socket-dir" annotation to template
 
 {{ if .Cluster.Features.Has "externalCloudProvider" }}
@@ -220,6 +220,6 @@ spec:
           emptyDir: {}
         - name: cloud-config
           secret:
-            secretName: cloud-config
+            secretName: cloud-config-csi
 {{ end }}
 {{ end }}

addons/csi/azure-disk/csi-azuredisk-node.yaml

@@ -17,7 +17,7 @@
 # Changes:
 # - image source includes registry templating
 # - removal of scheduling hints
-# - mount cloud-config instead of using host path
+# - mount cloud-config-csi instead of using host path
 
 {{ if .Cluster.Features.Has "externalCloudProvider" }}
 {{ if eq .Cluster.CloudProviderName "azure" }}
@@ -195,7 +195,7 @@ spec:
             type: Directory
           name: sys-class
         - secret:
-            secretName: cloud-config
+            secretName: cloud-config-csi
           name: cloud-config
 ---
 {{ end }}

addons/csi/azure-file/csi-azurefile-controller.yaml

@@ -18,6 +18,7 @@
 # - image source includes registry templating
 # - set replicas=1 (pod has a leader election)
 # - removal of tolerations
+# - mount cloud-config-csi for credentials
 # - add "cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes: socket-dir" annotation to template
 
 {{ if .Cluster.Features.Has "externalCloudProvider" }}
@@ -212,6 +213,6 @@ spec:
           emptyDir: {}
         - name: cloud-config
           secret:
-            secretName: cloud-config
+            secretName: cloud-config-csi
 {{ end }}
 {{ end }}

addons/csi/azure-file/csi-azurefile-node.yaml

@@ -17,7 +17,7 @@
 # Changes:
 # - image source includes registry templating
 # - removal of scheduling hints
-# - mount cloud-config instead of using host path
+# - mount cloud-config-csi instead of using host path
 
 {{ if .Cluster.Features.Has "externalCloudProvider" }}
 {{ if eq .Cluster.CloudProviderName "azure" }}
@@ -180,7 +180,7 @@ spec:
             type: Directory
           name: device-dir
         - secret:
-            secretName: cloud-config
+            secretName: cloud-config-csi
           name: cloud-config
 {{ end }}
 {{ end }}

addons/csi/openstack/controllerplugin.yaml

@@ -178,7 +178,7 @@ spec:
           emptyDir:
         - name: cloud-config
           secret:
-            secretName: cloud-config
+            secretName: cloud-config-csi
         - name: ca-bundle
           configMap:
             name: ca-bundle

addons/csi/openstack/nodeplugin.yaml

@@ -154,7 +154,7 @@ spec:
             type: Directory
         - name: secret-cinderplugin
           secret:
-            secretName: cloud-config
+            secretName: cloud-config-csi
         - name: ca-bundle
           configMap:
             name: ca-bundle

pkg/controller/seed-controller-manager/kubernetes/resources.go

@@ -471,7 +471,7 @@ func (r *Reconciler) GetSecretReconcilers(ctx context.Context, data *resources.T
 	namespace := data.Cluster().Status.NamespaceName
 
 	creators := []reconciling.NamedSecretReconcilerFactory{
-		cloudconfig.SecretReconciler(data),
+		cloudconfig.SecretReconciler(data, resources.CloudConfigSecretName),
 		certificates.RootCAReconciler(data),
 		certificates.FrontProxyCAReconciler(),
 		resources.ImagePullSecretReconciler(r.dockerPullConfigJSON),

pkg/controller/user-cluster-controller-manager/resources/cloudcontroller/deletion.go

@@ -0,0 +1,36 @@
+/*
+Copyright 2024 The Kubermatic Kubernetes Platform contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cloudcontroller
+
+import (
+	"k8c.io/kubermatic/v2/pkg/resources"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func ResourcesForDeletion(namespace string) []ctrlruntimeclient.Object {
+	return []ctrlruntimeclient.Object{
+		&corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      resources.CSICloudConfigSecretName,
+				Namespace: namespace,
+			},
+		},
+	}
+}

pkg/controller/user-cluster-controller-manager/resources/reconciler.go

@@ -83,15 +83,10 @@ func (r *reconciler) reconcile(ctx context.Context) error {
 	if err != nil {
 		return fmt.Errorf("failed to get userSSHKeys: %w", err)
 	}
-	cloudConfig, err := r.cloudConfig(ctx, resources.CloudConfigSeedSecretName)
-	if err != nil {
-		return fmt.Errorf("failed to get cloudConfig: %w", err)
-	}
 
 	data := reconcileData{
 		caCert:       caCert,
 		userSSHKeys:  userSSHKeys,
-		cloudConfig:  cloudConfig,
 		ccmMigration: r.ccmMigration || r.ccmMigrationCompleted,
 	}
 
@@ -101,11 +96,20 @@ func (r *reconciler) reconcile(ctx context.Context) error {
 	}
 
 	if !cluster.Spec.DisableCSIDriver {
-		if r.cloudProvider == kubermaticv1.VSphereCloudProvider || r.cloudProvider == kubermaticv1.VMwareCloudDirectorCloudProvider || (r.cloudProvider == kubermaticv1.NutanixCloudProvider && r.nutanixCSIEnabled) {
+		if r.cloudProvider == kubermaticv1.VSphereCloudProvider ||
+			r.cloudProvider == kubermaticv1.VMwareCloudDirectorCloudProvider ||
+			(r.cloudProvider == kubermaticv1.NutanixCloudProvider && r.nutanixCSIEnabled) {
 			data.csiCloudConfig, err = r.cloudConfig(ctx, resources.CSICloudConfigSecretName)
 			if err != nil {
 				return fmt.Errorf("failed to get csi config: %w", err)
 			}
+		} else if r.cloudProvider == kubermaticv1.AzureCloudProvider ||
+			r.cloudProvider == kubermaticv1.OpenstackCloudProvider {
+			// Azure and Openstack CSI drivers don't have dedicated CSI cloud config.
+			data.csiCloudConfig, err = r.cloudConfig(ctx, resources.CloudConfigSecretName)
+			if err != nil {
+				return fmt.Errorf("failed to get csi config: %w", err)
+			}
 		}
 	}
 
@@ -271,6 +275,12 @@ func (r *reconciler) reconcile(ctx context.Context) error {
 		}
 	}
 
+	if cluster.Spec.DisableCSIDriver {
+		if err := r.ensureCSIDriverResourcesAreRemoved(ctx); err != nil {
+			return err
+		}
+	}
+
 	// This code supports switching between OpenVPN and Konnectivity setup (in both directions).
 	// It can be removed one release after deprecating OpenVPN.
 	if r.isKonnectivityEnabled {
@@ -294,6 +304,7 @@ func (r *reconciler) reconcile(ctx context.Context) error {
 			return err
 		}
 	}
+
 	return nil
 }
 
@@ -872,9 +883,7 @@ func (r *reconciler) reconcileConfigMaps(ctx context.Context, data reconcileData
 }
 
 func (r *reconciler) reconcileSecrets(ctx context.Context, data reconcileData) error {
-	creators := []reconciling.NamedSecretReconcilerFactory{
-		cloudcontroller.CloudConfig(data.cloudConfig, resources.CloudConfigSecretName),
-	}
+	creators := []reconciling.NamedSecretReconcilerFactory{}
 	if !r.isKonnectivityEnabled {
 		creators = append(creators, openvpn.ClientCertificate(data.openVPNCACert))
 	} else {
@@ -887,9 +896,12 @@ func (r *reconciler) reconcileSecrets(ctx context.Context, data reconcileData) e
 	}
 
 	if data.csiCloudConfig != nil {
+		if r.cloudProvider == kubermaticv1.AzureCloudProvider || r.cloudProvider == kubermaticv1.OpenstackCloudProvider || r.cloudProvider == kubermaticv1.VSphereCloudProvider {
+			creators = append(creators, cloudcontroller.CloudConfig(data.csiCloudConfig, resources.CSICloudConfigSecretName))
+		}
+
 		if r.cloudProvider == kubermaticv1.VSphereCloudProvider {
-			creators = append(creators, cloudcontroller.CloudConfig(data.csiCloudConfig, resources.CSICloudConfigSecretName),
-				csisnapshotter.TLSServingCertificateReconciler(resources.CSISnapshotValidationWebhookName, data.caCert))
+			creators = append(creators, csisnapshotter.TLSServingCertificateReconciler(resources.CSISnapshotValidationWebhookName, data.caCert))
 			if data.ccmMigration {
 				creators = append(creators, csimigration.TLSServingCertificateReconciler(data.caCert))
 			}
@@ -1150,7 +1162,6 @@ type reconcileData struct {
 	openVPNCACert    *resources.ECDSAKeyPair
 	mlaGatewayCACert *resources.ECDSAKeyPair
 	userSSHKeys      map[string][]byte
-	cloudConfig      []byte
 	clusterVersion   semver.Semver
 	// csiCloudConfig is currently used only by vSphere, VMware Cloud Director and Nutanix,
 	// who need it to properly configure the external CSI driver; however this can be nil if the
@@ -1367,6 +1378,16 @@ func (r *reconciler) ensureMLAIsRemoved(ctx context.Context) error {
 	return nil
 }
 
+func (r *reconciler) ensureCSIDriverResourcesAreRemoved(ctx context.Context) error {
+	for _, resource := range cloudcontroller.ResourcesForDeletion(metav1.NamespaceSystem) {
+		err := r.Client.Delete(ctx, resource)
+		if err != nil && !apierrors.IsNotFound(err) {
+			return fmt.Errorf("failed to ensure CSI driver resources are removed/not present: %w", err)
+		}
+	}
+	return nil
+}
+
 func (r *reconciler) ensureOpenVPNSetupIsRemoved(ctx context.Context) error {
 	for _, resource := range openvpn.ResourcesForDeletion() {
 		err := r.Client.Delete(ctx, resource)

pkg/resources/cloudconfig/secret.go

@@ -44,9 +44,9 @@ type creatorData interface {
 }
 
 // SecretReconciler returns a function to create the Secret containing the cloud-config.
-func SecretReconciler(data creatorData) reconciling.NamedSecretReconcilerFactory {
+func SecretReconciler(data creatorData, name string) reconciling.NamedSecretReconcilerFactory {
 	return func() (string, reconciling.SecretReconciler) {
-		return resources.CloudConfigSecretName, func(cm *corev1.Secret) (*corev1.Secret, error) {
+		return name, func(cm *corev1.Secret) (*corev1.Secret, error) {
 			if cm.Data == nil {
 				cm.Data = map[string][]byte{}
 			}