Initial changes for multi vcenter support.

assets/csi_cloud_config.ini

@@ -3,8 +3,6 @@
 [Global]
 cluster-id = "${CLUSTER_ID}"
 
-[VirtualCenter "${VCENTER}"]
-insecure-flag = "true"
-datacenters = "${DATACENTERS}"
-migration-datastore-url = "${MIGRATION_DATASTORE_URL}"
+# Populate VCenters (multi) after here
+${VCENTERS}
 

assets/csi_cloud_config_vcenters.ini

@@ -0,0 +1,6 @@
+[VirtualCenter "${VCENTER}"]
+insecure-flag = "true"
+datacenters = "${DATACENTERS}"
+migration-datastore-url = "${MIGRATION_DATASTORE_URL}"
+password = "${PASSWORD}"
+user = "${USER}"

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/openshift/client-go v0.0.0-20240125160436-aa5df63097c4
 	github.com/openshift/library-go v0.0.0-20240130085015-2ad786549f07
 	github.com/prometheus/client_golang v1.18.0
+	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.0
 	github.com/vmware/govmomi v0.34.2
 	gopkg.in/gcfg.v1 v1.2.3
@@ -19,14 +20,15 @@ require (
 	k8s.io/apiextensions-apiserver v0.29.1
 	k8s.io/apimachinery v0.29.1
 	k8s.io/client-go v0.29.1
+	k8s.io/cloud-provider-vsphere v1.28.0
 	k8s.io/component-base v0.29.1
 	k8s.io/klog/v2 v2.120.1
 	k8s.io/legacy-cloud-providers v0.29.1
 )
 
 require (
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
-	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect
+	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230321174746-8dcc6526cfb1 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
@@ -71,7 +73,6 @@ require (
 	github.com/prometheus/common v0.46.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/robfig/cron v1.2.0 // indirect
-	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stoewer/go-strcase v1.3.0 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.12 // indirect

go.sum

@@ -5,8 +5,8 @@ cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGB
 cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
 github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
 github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
-github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df h1:7RFfzj4SSt6nnvCPbCqijJi1nWCd+TqAT3bYCStRC18=
-github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM=
+github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230321174746-8dcc6526cfb1 h1:X8MJ0fnN5FPdcGF5Ij2/OW+HgiJrRg3AfHAx1PJtIzM=
+github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230321174746-8dcc6526cfb1/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
 github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -332,6 +332,8 @@ k8s.io/client-go v0.29.1 h1:19B/+2NGEwnFLzt0uB5kNJnfTsbV8w6TgQRz9l7ti7A=
 k8s.io/client-go v0.29.1/go.mod h1:TDG/psL9hdet0TI9mGyHJSgRkW3H9JZk2dNEUS7bRks=
 k8s.io/cloud-provider v0.29.1 h1:bDLpOSpysWrtU2PCkvyP2sUTwRBa6MGCmxt68CRRW/8=
 k8s.io/cloud-provider v0.29.1/go.mod h1:u50Drm6AbuoKpsVbAstNiFHGgbSVHuJV4TWN5imdM2w=
+k8s.io/cloud-provider-vsphere v1.28.0 h1:S7KyDRI3EDQ5jFimTRNY+S0IcmLV+WpiLDafeAfVK5Y=
+k8s.io/cloud-provider-vsphere v1.28.0/go.mod h1:nGC4cRIj3uQkBM0gc9mIzknmepx1TjPJTSSpMZ8YKwM=
 k8s.io/component-base v0.29.1 h1:MUimqJPCRnnHsskTTjKD+IC1EHBbRCVyi37IoFBrkYw=
 k8s.io/component-base v0.29.1/go.mod h1:fP9GFjxYrLERq1GcWWZAE3bqbNcDKDytn2srWuHTtKc=
 k8s.io/component-helpers v0.29.1 h1:54MMEDu6xeJmMtAKztsPwu0kJKr4+jCUzaEIn2UXRoc=

pkg/operator/vspherecontroller/driver_starter.go

@@ -230,6 +230,7 @@ func WithVSphereCredentials(
 		}
 
 		// Add to csi-driver and vsphere-syncer containers the vSphere credentials, as env vars.
+		// TODO: NAG - With multi vCenter, this is unacceptable.  We can either set in ini, or find new way.
 		containers := deployment.Spec.Template.Spec.Containers
 		for i := range containers {
 			if containers[i].Name != "csi-driver" && containers[i].Name != "vsphere-syncer" {

pkg/operator/vspherecontroller/vspherecontroller.go

@@ -8,12 +8,12 @@ import (
 	"time"
 
 	"github.com/openshift/vmware-vsphere-csi-driver-operator/assets"
-	"gopkg.in/gcfg.v1"
 	iniv1 "gopkg.in/ini.v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
-	"k8s.io/legacy-cloud-providers/vsphere"
+	vsphere "k8s.io/cloud-provider-vsphere/pkg/common/config"
+	legacy "k8s.io/legacy-cloud-providers/vsphere"
 
 	"github.com/openshift/vmware-vsphere-csi-driver-operator/pkg/operator/storageclasscontroller"
 
@@ -460,9 +460,12 @@ func (c *VSphereController) createVCenterConnection(ctx context.Context, infra *
 	cfgString, ok := cloudConfigMap.Data[infra.Spec.CloudConfig.Key]
 	if !ok {
 		return fmt.Errorf("cloud config %s/%s does not contain key %q", cloudConfigNamespace, cloudConfig.Name, cloudConfig.Key)
-	}
-	cfg := new(vsphere.VSphereConfig)
-	err = gcfg.ReadStringInto(cfg, cfgString)
+	}*/
+	//cfg := new(vsphere.VSphereConfig)
+	//err = gcfg.ReadStringInto(cfg, cfgString)
+
+	// If we use infra to iterate through vcenters, do we need to load config?
+	/*cfg, err := vsphere.ReadConfig([]byte(cfgString))
 	if err != nil {
 		return err
 	}*/
@@ -484,7 +487,7 @@ func (c *VSphereController) createVCenterConnection(ctx context.Context, infra *
 		}
 
 		// just a hack for other function compatibility.
-		cfg := new(vsphere.VSphereConfig)
+		cfg := new(legacy.VSphereConfig)
 		cfg.Workspace.VCenterIP = vcenter.Server
 		cfg.Workspace.Datacenter = vcenter.Datacenters[0]
 		cfg.Workspace.DefaultDatastore = infra.Spec.PlatformSpec.VSphere.FailureDomains[0].Topology.Datastore
@@ -637,8 +640,12 @@ func (c *VSphereController) createCSIConfigMap(
 		return fmt.Errorf("cloud config %s/%s does not contain key %q", cloudConfigNamespace, cloudConfig.Name, cloudConfig.Key)
 	}
 
-	var cfg vsphere.VSphereConfig
-	err = gcfg.ReadStringInto(&cfg, cfgString)
+	// Update this to also support YAML.
+	//var cfg vsphere.VSphereConfig
+	//err = gcfg.ReadStringInto(&cfg, cfgString)
+
+	cfg, err := vsphere.ReadConfig([]byte(cfgString))
+
 	if err != nil {
 		return err
 	}
@@ -670,25 +677,44 @@ func (c *VSphereController) createCSIConfigMap(
 
 func (c *VSphereController) applyClusterCSIDriverChange(
 	infra *ocpv1.Infrastructure,
-	sourceCFG vsphere.VSphereConfig,
+	sourceCFG *vsphere.Config,
 	clusterCSIDriver *operatorapi.ClusterCSIDriver,
 	datastoreURL string) (*corev1.ConfigMap, error) {
 
 	csiConfigString := string(c.csiConfigManifest)
 
-	dataCenterNames, err := utils.GetDatacenters(&sourceCFG, c.multiVCenterEnabled)
+	csiVCenterConfigBytes, err := assets.ReadFile("csi_cloud_config_vcenters.ini")
 
 	if err != nil {
 		return nil, err
 	}
 
-	datacenters := strings.Join(dataCenterNames, ",")
+	// TODO: NAG - May need to look into new format here for multi vcenter support.
+	// Generate cluster id and append all vcenters.  Also need to inject user/pass for vcenters since driver does
+	// not support loading from secret.  It expect either in the ini file or as an ENV variable.
+
+	secret, err := c.secretLister.Secrets(c.targetNamespace).Get(cloudCredSecretName)
+
+	var vcenters string
+	for _, vcenter := range sourceCFG.VirtualCenter {
+		vcenterStr := string(csiVCenterConfigBytes)
+		user := string(secret.Data[vcenter.VCenterIP+".username"])
+		password := string(secret.Data[vcenter.VCenterIP+".password"])
+		for pattern, value := range map[string]string{
+			"${VCENTER}":                 vcenter.VCenterIP,
+			"${DATACENTERS}":             vcenter.Datacenters,
+			"${MIGRATION_DATASTORE_URL}": datastoreURL,
+			"${PASSWORD}":                password,
+			"${USER}":                    user,
+		} {
+			vcenterStr = strings.ReplaceAll(vcenterStr, pattern, value)
+		}
+		vcenters = vcenters + "\n" + vcenterStr
+	}
 
 	for pattern, value := range map[string]string{
-		"${CLUSTER_ID}":              infra.Status.InfrastructureName,
-		"${VCENTER}":                 sourceCFG.Workspace.VCenterIP,
-		"${DATACENTERS}":             datacenters,
-		"${MIGRATION_DATASTORE_URL}": datastoreURL,
+		"${CLUSTER_ID}": infra.Status.InfrastructureName,
+		"${VCENTERS}":   vcenters,
 	} {
 		csiConfigString = strings.ReplaceAll(csiConfigString, pattern, value)
 	}