openshift · openshift-merge-bot · Jun 15, 2024 · Jun 7, 2024
diff --git a/cmd/openshift-install/main.go b/cmd/openshift-install/main.go
@@ -13,6 +13,7 @@ import (
 	terminal "golang.org/x/term"
 	"k8s.io/klog"
 	klogv2 "k8s.io/klog/v2"
+	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 
 	"github.com/openshift/installer/cmd/openshift-install/command"
@@ -33,6 +34,8 @@ func main() {
 	fsv2.Set("stderrthreshold", "4")
 	klogv2.SetOutput(io.Discard)
 
+	ctrl.SetLogger(klogv2.Background())
+
 	installerMain()
 }
 

diff --git a/data/data/install.openshift.io_installconfigs.yaml b/data/data/install.openshift.io_installconfigs.yaml
@@ -4691,7 +4691,7 @@ spec:
                       - server
                       - user
                       type: object
-                    maxItems: 1
+                    maxItems: 3
                     minItems: 1
                     type: array
                 type: object

diff --git a/pkg/asset/cluster/vsphere/vsphere.go b/pkg/asset/cluster/vsphere/vsphere.go
@@ -9,12 +9,20 @@ import (
 func Metadata(config *types.InstallConfig) *typesvsphere.Metadata {
 	terraformPlatform := "vsphere"
 
-	// Since currently we only support a single vCenter
-	// just use the first entry in the VCenters slice.
-	return &typesvsphere.Metadata{
-		VCenter:           config.VSphere.VCenters[0].Server,
-		Username:          config.VSphere.VCenters[0].Username,
-		Password:          config.VSphere.VCenters[0].Password,
+	metadata := &typesvsphere.Metadata{
 		TerraformPlatform: terraformPlatform,
 	}
+
+	vcenterList := []typesvsphere.VCenters{}
+	for _, vcenter := range config.VSphere.VCenters {
+		vcenterDef := typesvsphere.VCenters{
+			VCenter:  vcenter.Server,
+			Username: vcenter.Username,
+			Password: vcenter.Password,
+		}
+		vcenterList = append(vcenterList, vcenterDef)
+	}
+	metadata.VCenters = vcenterList
+
+	return metadata
 }
diff --git a/pkg/asset/machines/vsphere/capimachines.go b/pkg/asset/machines/vsphere/capimachines.go
@@ -146,6 +146,16 @@ func GenerateMachines(ctx context.Context, clusterID string, config *types.Insta
 			Object: vsphereMachine,
 		})
 
+		// Need to determine the infrastructure ref since there may be multi vcenters.
+		clusterName := clusterID
+		for index, vcenter := range config.Platform.VSphere.VCenters {
+			if vcenter.Server == providerSpec.Workspace.Server {
+				clusterName = fmt.Sprintf("%v-%d", clusterID, index)
+				break
+			}
+		}
+
+		// Create capi machine for vspheremachine
 		machine := &capi.Machine{
 			ObjectMeta: metav1.ObjectMeta{
 				Namespace: capiutils.Namespace,
@@ -155,7 +165,7 @@ func GenerateMachines(ctx context.Context, clusterID string, config *types.Insta
 				},
 			},
 			Spec: capi.MachineSpec{
-				ClusterName: clusterID,
+				ClusterName: clusterName,
 				Bootstrap: capi.Bootstrap{
 					DataSecretName: ptr.To(fmt.Sprintf("%s-%s", clusterID, role)),
 				},
@@ -205,6 +215,15 @@ func GenerateMachines(ctx context.Context, clusterID string, config *types.Insta
 			Object: bootstrapVSphereMachine,
 		})
 
+		// Need to determine the infrastructure ref since there may be multi vcenters.
+		clusterName := clusterID
+		for index, vcenter := range config.Platform.VSphere.VCenters {
+			if vcenter.Server == bootstrapSpec.Server {
+				clusterName = fmt.Sprintf("%v-%d", clusterID, index)
+				break
+			}
+		}
+
 		bootstrapMachine := &capi.Machine{
 			ObjectMeta: metav1.ObjectMeta{
 				Name: bootstrapVSphereMachine.Name,
@@ -213,7 +232,7 @@ func GenerateMachines(ctx context.Context, clusterID string, config *types.Insta
 				},
 			},
 			Spec: capi.MachineSpec{
-				ClusterName: clusterID,
+				ClusterName: clusterName,
 				Bootstrap: capi.Bootstrap{
 					DataSecretName: ptr.To(fmt.Sprintf("%s-bootstrap", clusterID)),
 				},

diff --git a/pkg/asset/manifests/aws/cluster.go b/pkg/asset/manifests/aws/cluster.go
@@ -257,11 +257,13 @@ func GenerateClusterAssets(ic *installconfig.InstallConfig, clusterID *installco
 
 	return &capiutils.GenerateClusterAssetsOutput{
 		Manifests: manifests,
-		InfrastructureRef: &corev1.ObjectReference{
-			APIVersion: capa.GroupVersion.String(),
-			Kind:       "AWSCluster",
-			Name:       awsCluster.Name,
-			Namespace:  awsCluster.Namespace,
+		InfrastructureRefs: []*corev1.ObjectReference{
+			{
+				APIVersion: capa.GroupVersion.String(),
+				Kind:       "AWSCluster",
+				Name:       awsCluster.Name,
+				Namespace:  awsCluster.Namespace,
+			},
 		},
 	}, nil
 }

diff --git a/pkg/asset/manifests/azure/cluster.go b/pkg/asset/manifests/azure/cluster.go
@@ -175,11 +175,13 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID
 
 	return &capiutils.GenerateClusterAssetsOutput{
 		Manifests: manifests,
-		InfrastructureRef: &corev1.ObjectReference{
-			APIVersion: capz.GroupVersion.String(),
-			Kind:       "AzureCluster",
-			Name:       azureCluster.Name,
-			Namespace:  azureCluster.Namespace,
+		InfrastructureRefs: []*corev1.ObjectReference{
+			{
+				APIVersion: capz.GroupVersion.String(),
+				Kind:       "AzureCluster",
+				Name:       azureCluster.Name,
+				Namespace:  azureCluster.Namespace,
+			},
 		},
 	}, nil
 }
diff --git a/pkg/asset/manifests/capiutils/manifest.go b/pkg/asset/manifests/capiutils/manifest.go
@@ -13,8 +13,8 @@ const (
 
 // GenerateClusterAssetsOutput is the output of GenerateClusterAssets.
 type GenerateClusterAssetsOutput struct {
-	Manifests         []*asset.RuntimeFile
-	InfrastructureRef *corev1.ObjectReference
+	Manifests          []*asset.RuntimeFile
+	InfrastructureRefs []*corev1.ObjectReference
 }
 
 // GenerateMachinesOutput is the output of GenerateMachines.

diff --git a/pkg/asset/manifests/cloudproviderconfig.go b/pkg/asset/manifests/cloudproviderconfig.go
@@ -11,6 +11,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/yaml"
 
+	"github.com/openshift/api/features"
 	"github.com/openshift/installer/pkg/asset"
 	"github.com/openshift/installer/pkg/asset/installconfig"
 	ibmcloudmachines "github.com/openshift/installer/pkg/asset/machines/ibmcloud"
@@ -315,7 +316,15 @@ func (cpc *CloudProviderConfig) Generate(dependencies asset.Parents) error {
 		}
 		cm.Data[cloudProviderConfigDataKey] = powervsConfig
 	case vspheretypes.Name:
-		vsphereConfig, err := vspheremanifests.CloudProviderConfigIni(clusterID.InfraID, installConfig.Config.Platform.VSphere)
+		var vsphereConfig string
+		var err error
+		// When we GA multi vcenter, we should only support yaml generation here.
+		if installConfig.Config.EnabledFeatureGates().Enabled(features.FeatureGateVSphereMultiVCenters) {
+			vsphereConfig, err = vspheremanifests.CloudProviderConfigYaml(clusterID.InfraID, installConfig.Config.Platform.VSphere)
+		} else {
+			vsphereConfig, err = vspheremanifests.CloudProviderConfigIni(clusterID.InfraID, installConfig.Config.Platform.VSphere)
+		}
+
 		if err != nil {
 			return errors.Wrap(err, "could not create cloud provider config")
 		}

diff --git a/pkg/asset/manifests/clusterapi/cluster.go b/pkg/asset/manifests/clusterapi/cluster.go
@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -86,20 +87,6 @@ func (c *Cluster) Generate(dependencies asset.Parents) error {
 	namespace.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Namespace"))
 	c.FileList = append(c.FileList, &asset.RuntimeFile{Object: namespace, File: asset.File{Filename: "000_capi-namespace.yaml"}})
 
-	cluster := &clusterv1.Cluster{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      clusterID.InfraID,
-			Namespace: capiutils.Namespace,
-		},
-		Spec: clusterv1.ClusterSpec{
-			ClusterNetwork: &clusterv1.ClusterNetwork{
-				APIServerPort: ptr.To[int32](6443),
-			},
-		},
-	}
-	cluster.SetGroupVersionKind(clusterv1.GroupVersion.WithKind("Cluster"))
-	c.FileList = append(c.FileList, &asset.RuntimeFile{Object: cluster, File: asset.File{Filename: "01_capi-cluster.yaml"}})
-
 	var out *capiutils.GenerateClusterAssetsOutput
 	switch platform := installConfig.Config.Platform.Name(); platform {
 	case awstypes.Name:
@@ -149,12 +136,28 @@ func (c *Cluster) Generate(dependencies asset.Parents) error {
 		return fmt.Errorf("unsupported platform %q", platform)
 	}
 
-	// Set the infrastructure reference in the Cluster object.
-	cluster.Spec.InfrastructureRef = out.InfrastructureRef
-	if cluster.Spec.InfrastructureRef == nil {
+	if len(out.InfrastructureRefs) == 0 {
 		return fmt.Errorf("failed to generate manifests: cluster.Spec.InfrastructureRef was never set")
 	}
 
+	logrus.Infof("Adding clusters...")
+	for index, infra := range out.InfrastructureRefs {
+		cluster := &clusterv1.Cluster{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      infra.Name,
+				Namespace: capiutils.Namespace,
+			},
+			Spec: clusterv1.ClusterSpec{
+				ClusterNetwork: &clusterv1.ClusterNetwork{
+					APIServerPort: ptr.To[int32](6443),
+				},
+			},
+		}
+		cluster.Spec.InfrastructureRef = infra
+		cluster.SetGroupVersionKind(clusterv1.GroupVersion.WithKind("Cluster"))
+		c.FileList = append(c.FileList, &asset.RuntimeFile{Object: cluster, File: asset.File{Filename: fmt.Sprintf("01_capi-cluster-%d.yaml", index)}})
+	}
+
 	// Append the infrastructure manifests.
 	c.FileList = append(c.FileList, out.Manifests...)
 

diff --git a/pkg/asset/manifests/gcp/cluster.go b/pkg/asset/manifests/gcp/cluster.go
@@ -133,11 +133,13 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID
 
 	return &capiutils.GenerateClusterAssetsOutput{
 		Manifests: manifests,
-		InfrastructureRef: &corev1.ObjectReference{
-			APIVersion: capg.GroupVersion.String(),
-			Kind:       "GCPCluster",
-			Name:       gcpCluster.Name,
-			Namespace:  gcpCluster.Namespace,
+		InfrastructureRefs: []*corev1.ObjectReference{
+			{
+				APIVersion: capg.GroupVersion.String(),
+				Kind:       "GCPCluster",
+				Name:       gcpCluster.Name,
+				Namespace:  gcpCluster.Namespace,
+			},
 		},
 	}, nil
 }

diff --git a/pkg/asset/manifests/nutanix/cluster.go b/pkg/asset/manifests/nutanix/cluster.go
@@ -102,11 +102,13 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID
 
 	return &capiutils.GenerateClusterAssetsOutput{
 		Manifests: manifests,
-		InfrastructureRef: &corev1.ObjectReference{
-			APIVersion: "infrastructure.cluster.x-k8s.io/v1beta1",
-			Kind:       "NutanixCluster",
-			Name:       clusterID.InfraID,
-			Namespace:  capiutils.Namespace,
+		InfrastructureRefs: []*corev1.ObjectReference{
+			{
+				APIVersion: "infrastructure.cluster.x-k8s.io/v1beta1",
+				Kind:       "NutanixCluster",
+				Name:       clusterID.InfraID,
+				Namespace:  capiutils.Namespace,
+			},
 		},
 	}, nil
 }
diff --git a/pkg/asset/manifests/openstack/cluster.go b/pkg/asset/manifests/openstack/cluster.go
@@ -116,11 +116,13 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID
 
 	return &capiutils.GenerateClusterAssetsOutput{
 		Manifests: manifests,
-		InfrastructureRef: &corev1.ObjectReference{
-			APIVersion: capo.GroupVersion.String(),
-			Kind:       "OpenStackCluster",
-			Name:       openStackCluster.Name,
-			Namespace:  openStackCluster.Namespace,
+		InfrastructureRefs: []*corev1.ObjectReference{
+			{
+				APIVersion: capo.GroupVersion.String(),
+				Kind:       "OpenStackCluster",
+				Name:       openStackCluster.Name,
+				Namespace:  openStackCluster.Namespace,
+			},
 		},
 	}, nil
 }

diff --git a/pkg/asset/manifests/powervs/cluster.go b/pkg/asset/manifests/powervs/cluster.go
@@ -225,11 +225,13 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID
 
 	return &capiutils.GenerateClusterAssetsOutput{
 		Manifests: manifests,
-		InfrastructureRef: &corev1.ObjectReference{
-			APIVersion: "infrastructure.cluster.x-k8s.io/v1beta2",
-			Kind:       "IBMPowerVSCluster",
-			Name:       powerVSCluster.Name,
-			Namespace:  powerVSCluster.Namespace,
+		InfrastructureRefs: []*corev1.ObjectReference{
+			{
+				APIVersion: "infrastructure.cluster.x-k8s.io/v1beta2",
+				Kind:       "IBMPowerVSCluster",
+				Name:       powerVSCluster.Name,
+				Namespace:  powerVSCluster.Namespace,
+			},
 		},
 	}, nil
 }
diff --git a/pkg/asset/manifests/vsphere/cloudproviderconfig.go b/pkg/asset/manifests/vsphere/cloudproviderconfig.go
@@ -32,9 +32,10 @@ func CloudProviderConfigYaml(infraID string, p *vspheretypes.Platform) (string,
 			vCenterPort = vCenter.Port
 		}
 		vCenterConfig := cloudconfig.VirtualCenterConfigYAML{
-			VCenterIP:   vCenter.Server,
-			VCenterPort: uint(vCenterPort),
-			Datacenters: vCenter.Datacenters,
+			VCenterIP:    vCenter.Server,
+			VCenterPort:  uint(vCenterPort),
+			Datacenters:  vCenter.Datacenters,
+			InsecureFlag: true,
 		}
 		vCenters[vCenter.Server] = &vCenterConfig
 	}
@@ -43,6 +44,7 @@ func CloudProviderConfigYaml(infraID string, p *vspheretypes.Platform) (string,
 		Global: cloudconfig.GlobalYAML{
 			SecretName:      "vsphere-creds",
 			SecretNamespace: "kube-system",
+			InsecureFlag:    true,
 		},
 		Vcenter: vCenters,
 		Labels: cloudconfig.LabelsYAML{

diff --git a/pkg/asset/manifests/vsphere/cloudproviderconfig_test.go b/pkg/asset/manifests/vsphere/cloudproviderconfig_test.go
@@ -39,7 +39,7 @@ zone = "openshift-zone"
   password: ""
   server: ""
   port: 0
-  insecureFlag: false
+  insecureFlag: true
   datacenters: []
   soapRoundtripCount: 0
   caFile: ""
@@ -57,7 +57,7 @@ vcenter:
     tenantref: ""
     server: test-vcenter
     port: 443
-    insecureFlag: false
+    insecureFlag: true
     datacenters:
     - test-datacenter
     - test-datacenter2