diff --git a/cmd/openshift-install/main.go b/cmd/openshift-install/main.go index b147ea604b8..17eb0cbe344 100644 --- a/cmd/openshift-install/main.go +++ b/cmd/openshift-install/main.go @@ -13,6 +13,7 @@ import ( terminal "golang.org/x/term" "k8s.io/klog" klogv2 "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/manager/signals" "github.com/openshift/installer/cmd/openshift-install/command" @@ -33,6 +34,8 @@ func main() { fsv2.Set("stderrthreshold", "4") klogv2.SetOutput(io.Discard) + ctrl.SetLogger(klogv2.Background()) + installerMain() } diff --git a/data/data/install.openshift.io_installconfigs.yaml b/data/data/install.openshift.io_installconfigs.yaml index a8befaa0f5e..412fb693e0d 100644 --- a/data/data/install.openshift.io_installconfigs.yaml +++ b/data/data/install.openshift.io_installconfigs.yaml @@ -4691,7 +4691,7 @@ spec: - server - user type: object - maxItems: 1 + maxItems: 3 minItems: 1 type: array type: object diff --git a/pkg/asset/cluster/vsphere/vsphere.go b/pkg/asset/cluster/vsphere/vsphere.go index 4db6ca824f0..d05820852b5 100644 --- a/pkg/asset/cluster/vsphere/vsphere.go +++ b/pkg/asset/cluster/vsphere/vsphere.go @@ -9,12 +9,20 @@ import ( func Metadata(config *types.InstallConfig) *typesvsphere.Metadata { terraformPlatform := "vsphere" - // Since currently we only support a single vCenter - // just use the first entry in the VCenters slice. - return &typesvsphere.Metadata{ - VCenter: config.VSphere.VCenters[0].Server, - Username: config.VSphere.VCenters[0].Username, - Password: config.VSphere.VCenters[0].Password, + metadata := &typesvsphere.Metadata{ TerraformPlatform: terraformPlatform, } + + vcenterList := []typesvsphere.VCenters{} + for _, vcenter := range config.VSphere.VCenters { + vcenterDef := typesvsphere.VCenters{ + VCenter: vcenter.Server, + Username: vcenter.Username, + Password: vcenter.Password, + } + vcenterList = append(vcenterList, vcenterDef) + } + metadata.VCenters = vcenterList + + return metadata } diff --git a/pkg/asset/machines/vsphere/capimachines.go b/pkg/asset/machines/vsphere/capimachines.go index 37ee10f0463..be51b0ce817 100644 --- a/pkg/asset/machines/vsphere/capimachines.go +++ b/pkg/asset/machines/vsphere/capimachines.go @@ -146,6 +146,16 @@ func GenerateMachines(ctx context.Context, clusterID string, config *types.Insta Object: vsphereMachine, }) + // Need to determine the infrastructure ref since there may be multi vcenters. + clusterName := clusterID + for index, vcenter := range config.Platform.VSphere.VCenters { + if vcenter.Server == providerSpec.Workspace.Server { + clusterName = fmt.Sprintf("%v-%d", clusterID, index) + break + } + } + + // Create capi machine for vspheremachine machine := &capi.Machine{ ObjectMeta: metav1.ObjectMeta{ Namespace: capiutils.Namespace, @@ -155,7 +165,7 @@ func GenerateMachines(ctx context.Context, clusterID string, config *types.Insta }, }, Spec: capi.MachineSpec{ - ClusterName: clusterID, + ClusterName: clusterName, Bootstrap: capi.Bootstrap{ DataSecretName: ptr.To(fmt.Sprintf("%s-%s", clusterID, role)), }, @@ -205,6 +215,15 @@ func GenerateMachines(ctx context.Context, clusterID string, config *types.Insta Object: bootstrapVSphereMachine, }) + // Need to determine the infrastructure ref since there may be multi vcenters. + clusterName := clusterID + for index, vcenter := range config.Platform.VSphere.VCenters { + if vcenter.Server == bootstrapSpec.Server { + clusterName = fmt.Sprintf("%v-%d", clusterID, index) + break + } + } + bootstrapMachine := &capi.Machine{ ObjectMeta: metav1.ObjectMeta{ Name: bootstrapVSphereMachine.Name, @@ -213,7 +232,7 @@ func GenerateMachines(ctx context.Context, clusterID string, config *types.Insta }, }, Spec: capi.MachineSpec{ - ClusterName: clusterID, + ClusterName: clusterName, Bootstrap: capi.Bootstrap{ DataSecretName: ptr.To(fmt.Sprintf("%s-bootstrap", clusterID)), }, diff --git a/pkg/asset/manifests/aws/cluster.go b/pkg/asset/manifests/aws/cluster.go index 6e4b21e620e..825d8e0c8e3 100644 --- a/pkg/asset/manifests/aws/cluster.go +++ b/pkg/asset/manifests/aws/cluster.go @@ -257,11 +257,13 @@ func GenerateClusterAssets(ic *installconfig.InstallConfig, clusterID *installco return &capiutils.GenerateClusterAssetsOutput{ Manifests: manifests, - InfrastructureRef: &corev1.ObjectReference{ - APIVersion: capa.GroupVersion.String(), - Kind: "AWSCluster", - Name: awsCluster.Name, - Namespace: awsCluster.Namespace, + InfrastructureRefs: []*corev1.ObjectReference{ + { + APIVersion: capa.GroupVersion.String(), + Kind: "AWSCluster", + Name: awsCluster.Name, + Namespace: awsCluster.Namespace, + }, }, }, nil } diff --git a/pkg/asset/manifests/azure/cluster.go b/pkg/asset/manifests/azure/cluster.go index c82192408f7..c092f98ab22 100644 --- a/pkg/asset/manifests/azure/cluster.go +++ b/pkg/asset/manifests/azure/cluster.go @@ -175,11 +175,13 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID return &capiutils.GenerateClusterAssetsOutput{ Manifests: manifests, - InfrastructureRef: &corev1.ObjectReference{ - APIVersion: capz.GroupVersion.String(), - Kind: "AzureCluster", - Name: azureCluster.Name, - Namespace: azureCluster.Namespace, + InfrastructureRefs: []*corev1.ObjectReference{ + { + APIVersion: capz.GroupVersion.String(), + Kind: "AzureCluster", + Name: azureCluster.Name, + Namespace: azureCluster.Namespace, + }, }, }, nil } diff --git a/pkg/asset/manifests/capiutils/manifest.go b/pkg/asset/manifests/capiutils/manifest.go index f21191c0425..fba95cb63b3 100644 --- a/pkg/asset/manifests/capiutils/manifest.go +++ b/pkg/asset/manifests/capiutils/manifest.go @@ -13,8 +13,8 @@ const ( // GenerateClusterAssetsOutput is the output of GenerateClusterAssets. type GenerateClusterAssetsOutput struct { - Manifests []*asset.RuntimeFile - InfrastructureRef *corev1.ObjectReference + Manifests []*asset.RuntimeFile + InfrastructureRefs []*corev1.ObjectReference } // GenerateMachinesOutput is the output of GenerateMachines. diff --git a/pkg/asset/manifests/cloudproviderconfig.go b/pkg/asset/manifests/cloudproviderconfig.go index d2adcdcbca1..2a7f0b31f0f 100644 --- a/pkg/asset/manifests/cloudproviderconfig.go +++ b/pkg/asset/manifests/cloudproviderconfig.go @@ -11,6 +11,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/yaml" + "github.com/openshift/api/features" "github.com/openshift/installer/pkg/asset" "github.com/openshift/installer/pkg/asset/installconfig" ibmcloudmachines "github.com/openshift/installer/pkg/asset/machines/ibmcloud" @@ -315,7 +316,15 @@ func (cpc *CloudProviderConfig) Generate(dependencies asset.Parents) error { } cm.Data[cloudProviderConfigDataKey] = powervsConfig case vspheretypes.Name: - vsphereConfig, err := vspheremanifests.CloudProviderConfigIni(clusterID.InfraID, installConfig.Config.Platform.VSphere) + var vsphereConfig string + var err error + // When we GA multi vcenter, we should only support yaml generation here. + if installConfig.Config.EnabledFeatureGates().Enabled(features.FeatureGateVSphereMultiVCenters) { + vsphereConfig, err = vspheremanifests.CloudProviderConfigYaml(clusterID.InfraID, installConfig.Config.Platform.VSphere) + } else { + vsphereConfig, err = vspheremanifests.CloudProviderConfigIni(clusterID.InfraID, installConfig.Config.Platform.VSphere) + } + if err != nil { return errors.Wrap(err, "could not create cloud provider config") } diff --git a/pkg/asset/manifests/clusterapi/cluster.go b/pkg/asset/manifests/clusterapi/cluster.go index bdb2988600e..b6189d62a4a 100644 --- a/pkg/asset/manifests/clusterapi/cluster.go +++ b/pkg/asset/manifests/clusterapi/cluster.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/pkg/errors" + "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -86,20 +87,6 @@ func (c *Cluster) Generate(dependencies asset.Parents) error { namespace.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Namespace")) c.FileList = append(c.FileList, &asset.RuntimeFile{Object: namespace, File: asset.File{Filename: "000_capi-namespace.yaml"}}) - cluster := &clusterv1.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: clusterID.InfraID, - Namespace: capiutils.Namespace, - }, - Spec: clusterv1.ClusterSpec{ - ClusterNetwork: &clusterv1.ClusterNetwork{ - APIServerPort: ptr.To[int32](6443), - }, - }, - } - cluster.SetGroupVersionKind(clusterv1.GroupVersion.WithKind("Cluster")) - c.FileList = append(c.FileList, &asset.RuntimeFile{Object: cluster, File: asset.File{Filename: "01_capi-cluster.yaml"}}) - var out *capiutils.GenerateClusterAssetsOutput switch platform := installConfig.Config.Platform.Name(); platform { case awstypes.Name: @@ -149,12 +136,28 @@ func (c *Cluster) Generate(dependencies asset.Parents) error { return fmt.Errorf("unsupported platform %q", platform) } - // Set the infrastructure reference in the Cluster object. - cluster.Spec.InfrastructureRef = out.InfrastructureRef - if cluster.Spec.InfrastructureRef == nil { + if len(out.InfrastructureRefs) == 0 { return fmt.Errorf("failed to generate manifests: cluster.Spec.InfrastructureRef was never set") } + logrus.Infof("Adding clusters...") + for index, infra := range out.InfrastructureRefs { + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: infra.Name, + Namespace: capiutils.Namespace, + }, + Spec: clusterv1.ClusterSpec{ + ClusterNetwork: &clusterv1.ClusterNetwork{ + APIServerPort: ptr.To[int32](6443), + }, + }, + } + cluster.Spec.InfrastructureRef = infra + cluster.SetGroupVersionKind(clusterv1.GroupVersion.WithKind("Cluster")) + c.FileList = append(c.FileList, &asset.RuntimeFile{Object: cluster, File: asset.File{Filename: fmt.Sprintf("01_capi-cluster-%d.yaml", index)}}) + } + // Append the infrastructure manifests. c.FileList = append(c.FileList, out.Manifests...) diff --git a/pkg/asset/manifests/gcp/cluster.go b/pkg/asset/manifests/gcp/cluster.go index 347ac22f19b..328e0f35c0a 100644 --- a/pkg/asset/manifests/gcp/cluster.go +++ b/pkg/asset/manifests/gcp/cluster.go @@ -133,11 +133,13 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID return &capiutils.GenerateClusterAssetsOutput{ Manifests: manifests, - InfrastructureRef: &corev1.ObjectReference{ - APIVersion: capg.GroupVersion.String(), - Kind: "GCPCluster", - Name: gcpCluster.Name, - Namespace: gcpCluster.Namespace, + InfrastructureRefs: []*corev1.ObjectReference{ + { + APIVersion: capg.GroupVersion.String(), + Kind: "GCPCluster", + Name: gcpCluster.Name, + Namespace: gcpCluster.Namespace, + }, }, }, nil } diff --git a/pkg/asset/manifests/nutanix/cluster.go b/pkg/asset/manifests/nutanix/cluster.go index 70527cda93b..acbd72a147e 100644 --- a/pkg/asset/manifests/nutanix/cluster.go +++ b/pkg/asset/manifests/nutanix/cluster.go @@ -102,11 +102,13 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID return &capiutils.GenerateClusterAssetsOutput{ Manifests: manifests, - InfrastructureRef: &corev1.ObjectReference{ - APIVersion: "infrastructure.cluster.x-k8s.io/v1beta1", - Kind: "NutanixCluster", - Name: clusterID.InfraID, - Namespace: capiutils.Namespace, + InfrastructureRefs: []*corev1.ObjectReference{ + { + APIVersion: "infrastructure.cluster.x-k8s.io/v1beta1", + Kind: "NutanixCluster", + Name: clusterID.InfraID, + Namespace: capiutils.Namespace, + }, }, }, nil } diff --git a/pkg/asset/manifests/openstack/cluster.go b/pkg/asset/manifests/openstack/cluster.go index 692cf22e828..4b0c7f56efd 100644 --- a/pkg/asset/manifests/openstack/cluster.go +++ b/pkg/asset/manifests/openstack/cluster.go @@ -116,11 +116,13 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID return &capiutils.GenerateClusterAssetsOutput{ Manifests: manifests, - InfrastructureRef: &corev1.ObjectReference{ - APIVersion: capo.GroupVersion.String(), - Kind: "OpenStackCluster", - Name: openStackCluster.Name, - Namespace: openStackCluster.Namespace, + InfrastructureRefs: []*corev1.ObjectReference{ + { + APIVersion: capo.GroupVersion.String(), + Kind: "OpenStackCluster", + Name: openStackCluster.Name, + Namespace: openStackCluster.Namespace, + }, }, }, nil } diff --git a/pkg/asset/manifests/powervs/cluster.go b/pkg/asset/manifests/powervs/cluster.go index 92461c6eaa9..6e0d733ca75 100644 --- a/pkg/asset/manifests/powervs/cluster.go +++ b/pkg/asset/manifests/powervs/cluster.go @@ -225,11 +225,13 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID return &capiutils.GenerateClusterAssetsOutput{ Manifests: manifests, - InfrastructureRef: &corev1.ObjectReference{ - APIVersion: "infrastructure.cluster.x-k8s.io/v1beta2", - Kind: "IBMPowerVSCluster", - Name: powerVSCluster.Name, - Namespace: powerVSCluster.Namespace, + InfrastructureRefs: []*corev1.ObjectReference{ + { + APIVersion: "infrastructure.cluster.x-k8s.io/v1beta2", + Kind: "IBMPowerVSCluster", + Name: powerVSCluster.Name, + Namespace: powerVSCluster.Namespace, + }, }, }, nil } diff --git a/pkg/asset/manifests/vsphere/cloudproviderconfig.go b/pkg/asset/manifests/vsphere/cloudproviderconfig.go index 33b0f38473b..b79fd620298 100644 --- a/pkg/asset/manifests/vsphere/cloudproviderconfig.go +++ b/pkg/asset/manifests/vsphere/cloudproviderconfig.go @@ -32,9 +32,10 @@ func CloudProviderConfigYaml(infraID string, p *vspheretypes.Platform) (string, vCenterPort = vCenter.Port } vCenterConfig := cloudconfig.VirtualCenterConfigYAML{ - VCenterIP: vCenter.Server, - VCenterPort: uint(vCenterPort), - Datacenters: vCenter.Datacenters, + VCenterIP: vCenter.Server, + VCenterPort: uint(vCenterPort), + Datacenters: vCenter.Datacenters, + InsecureFlag: true, } vCenters[vCenter.Server] = &vCenterConfig } @@ -43,6 +44,7 @@ func CloudProviderConfigYaml(infraID string, p *vspheretypes.Platform) (string, Global: cloudconfig.GlobalYAML{ SecretName: "vsphere-creds", SecretNamespace: "kube-system", + InsecureFlag: true, }, Vcenter: vCenters, Labels: cloudconfig.LabelsYAML{ diff --git a/pkg/asset/manifests/vsphere/cloudproviderconfig_test.go b/pkg/asset/manifests/vsphere/cloudproviderconfig_test.go index dc650a7d96c..63770eb17e9 100644 --- a/pkg/asset/manifests/vsphere/cloudproviderconfig_test.go +++ b/pkg/asset/manifests/vsphere/cloudproviderconfig_test.go @@ -39,7 +39,7 @@ zone = "openshift-zone" password: "" server: "" port: 0 - insecureFlag: false + insecureFlag: true datacenters: [] soapRoundtripCount: 0 caFile: "" @@ -57,7 +57,7 @@ vcenter: tenantref: "" server: test-vcenter port: 443 - insecureFlag: false + insecureFlag: true datacenters: - test-datacenter - test-datacenter2 diff --git a/pkg/asset/manifests/vsphere/cluster.go b/pkg/asset/manifests/vsphere/cluster.go index b13e1dfd39c..e4873fb44b7 100644 --- a/pkg/asset/manifests/vsphere/cluster.go +++ b/pkg/asset/manifests/vsphere/cluster.go @@ -16,55 +16,60 @@ import ( func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID *installconfig.ClusterID) (*capiutils.GenerateClusterAssetsOutput, error) { manifests := []*asset.RuntimeFile{} - vcenter := installConfig.Config.VSphere.VCenters[0] + assetOutput := &capiutils.GenerateClusterAssetsOutput{} - vsphereCreds := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "vsphere-creds", - Namespace: capiutils.Namespace, - }, - Data: make(map[string][]byte), - } - vsphereCreds.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Secret")) + for index, vcenter := range installConfig.Config.VSphere.VCenters { + vsphereCreds := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("vsphere-creds-%d", index), + Namespace: capiutils.Namespace, + }, + Data: make(map[string][]byte), + } + vsphereCreds.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Secret")) - vsphereCreds.Data["username"] = []byte(vcenter.Username) - vsphereCreds.Data["password"] = []byte(vcenter.Password) + vsphereCreds.Data["username"] = []byte(vcenter.Username) + vsphereCreds.Data["password"] = []byte(vcenter.Password) - manifests = append(manifests, &asset.RuntimeFile{ - Object: vsphereCreds, - File: asset.File{Filename: "01_vsphere-creds.yaml"}, - }) + manifests = append(manifests, &asset.RuntimeFile{ + Object: vsphereCreds, + File: asset.File{Filename: fmt.Sprintf("01_%v.yaml", vsphereCreds.Name)}, + }) - vsphereCluster := &capv.VSphereCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: clusterID.InfraID, - Namespace: capiutils.Namespace, - }, - Spec: capv.VSphereClusterSpec{ - Server: fmt.Sprintf("https://%s", vcenter.Server), - ControlPlaneEndpoint: capv.APIEndpoint{ - Host: fmt.Sprintf("api.%s.%s", installConfig.Config.ObjectMeta.Name, installConfig.Config.BaseDomain), - Port: 6443, + vsphereCluster := &capv.VSphereCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%v-%d", clusterID.InfraID, index), + Namespace: capiutils.Namespace, }, - IdentityRef: &capv.VSphereIdentityReference{ - Kind: capv.SecretKind, - Name: "vsphere-creds", + Spec: capv.VSphereClusterSpec{ + Server: fmt.Sprintf("https://%s", vcenter.Server), + ControlPlaneEndpoint: capv.APIEndpoint{ + Host: fmt.Sprintf("api.%s.%s", installConfig.Config.ObjectMeta.Name, installConfig.Config.BaseDomain), + Port: 6443, + }, + IdentityRef: &capv.VSphereIdentityReference{ + Kind: capv.SecretKind, + Name: vsphereCreds.Name, + }, }, - }, - } - vsphereCluster.SetGroupVersionKind(capv.GroupVersion.WithKind("VSphereCluster")) - manifests = append(manifests, &asset.RuntimeFile{ - Object: vsphereCluster, - File: asset.File{Filename: "01_vsphere-cluster.yaml"}, - }) + } + vsphereCluster.SetGroupVersionKind(capv.GroupVersion.WithKind("VSphereCluster")) + manifests = append(manifests, &asset.RuntimeFile{ + Object: vsphereCluster, + File: asset.File{Filename: fmt.Sprintf("01_vsphere-cluster-%d.yaml", index)}, + }) - return &capiutils.GenerateClusterAssetsOutput{ - Manifests: manifests, - InfrastructureRef: &corev1.ObjectReference{ - APIVersion: capv.GroupVersion.String(), + infra := &corev1.ObjectReference{ + APIVersion: "infrastructure.cluster.x-k8s.io/v1beta1", Kind: "VSphereCluster", - Name: clusterID.InfraID, + Name: vsphereCluster.Name, Namespace: capiutils.Namespace, - }, - }, nil + } + + assetOutput.InfrastructureRefs = append(assetOutput.InfrastructureRefs, infra) + } + + assetOutput.Manifests = manifests + + return assetOutput, nil } diff --git a/pkg/destroy/vsphere/vsphere.go b/pkg/destroy/vsphere/vsphere.go index 90383513d69..777ac9a813d 100644 --- a/pkg/destroy/vsphere/vsphere.go +++ b/pkg/destroy/vsphere/vsphere.go @@ -22,27 +22,42 @@ type ClusterUninstaller struct { InfraID string terraformPlatform string - Logger logrus.FieldLogger - client API + Logger logrus.FieldLogger + clients []API } // New returns an VSphere destroyer from ClusterMetadata. func New(logger logrus.FieldLogger, metadata *installertypes.ClusterMetadata) (providers.Destroyer, error) { - client, err := NewClient(metadata.VSphere.VCenter, metadata.VSphere.Username, metadata.VSphere.Password) - if err != nil { - return nil, err + var clients []API + + // We have two ways of processing metadata. Older metadata has only 1 vcenter but configured at root level. New + // way is for all vcenter data to be part of the vcenters array. + if len(metadata.VSphere.VCenters) > 0 { + for _, vsphere := range metadata.VSphere.VCenters { + client, err := NewClient(vsphere.VCenter, vsphere.Username, vsphere.Password) + if err != nil { + return nil, err + } + clients = append(clients, client) + } + } else { + client, err := NewClient(metadata.VSphere.VCenter, metadata.VSphere.Username, metadata.VSphere.Password) + if err != nil { + return nil, err + } + clients = append(clients, client) } - return newWithClient(logger, metadata, client), nil + return newWithClient(logger, metadata, clients), nil } -func newWithClient(logger logrus.FieldLogger, metadata *installertypes.ClusterMetadata, client API) *ClusterUninstaller { +func newWithClient(logger logrus.FieldLogger, metadata *installertypes.ClusterMetadata, clients []API) *ClusterUninstaller { return &ClusterUninstaller{ ClusterID: metadata.ClusterID, InfraID: metadata.InfraID, terraformPlatform: metadata.VSphere.TerraformPlatform, - Logger: logger, - client: client, + Logger: logger, + clients: clients, } } @@ -52,34 +67,36 @@ func (o *ClusterUninstaller) deleteFolder(ctx context.Context) error { o.Logger.Debug("Delete Folder") - folderMoList, err := o.client.ListFolders(ctx, o.InfraID) - if err != nil { - return err - } - - if len(folderMoList) == 0 { - o.Logger.Debug("All folders deleted") - return nil - } + for _, client := range o.clients { + folderMoList, err := client.ListFolders(ctx, o.InfraID) + if err != nil { + return err + } - // If there are no children in the folder, go ahead and remove it + if len(folderMoList) == 0 { + o.Logger.Debug("All folders deleted") + return nil + } - for _, f := range folderMoList { - folderLogger := o.Logger.WithField("Folder", f.Name) - if numChildren := len(f.ChildEntity); numChildren > 0 { - entities := make([]string, 0, numChildren) - for _, child := range f.ChildEntity { - entities = append(entities, fmt.Sprintf("%s:%s", child.Type, child.Value)) + // If there are no children in the folder, go ahead and remove it + + for _, f := range folderMoList { + folderLogger := o.Logger.WithField("Folder", f.Name) + if numChildren := len(f.ChildEntity); numChildren > 0 { + entities := make([]string, 0, numChildren) + for _, child := range f.ChildEntity { + entities = append(entities, fmt.Sprintf("%s:%s", child.Type, child.Value)) + } + folderLogger.Errorf("Folder should be empty but contains %d objects: %s. The installer will retry removing \"virtualmachine\" objects, but any other type will need to be removed manually before the deprovision can proceed", numChildren, strings.Join(entities, ", ")) + return errors.Errorf("Expected Folder %s to be empty", f.Name) } - folderLogger.Errorf("Folder should be empty but contains %d objects: %s. The installer will retry removing \"virtualmachine\" objects, but any other type will need to be removed manually before the deprovision can proceed", numChildren, strings.Join(entities, ", ")) - return errors.Errorf("Expected Folder %s to be empty", f.Name) - } - err = o.client.DeleteFolder(ctx, f) - if err != nil { - folderLogger.Debug(err) - return err + err = client.DeleteFolder(ctx, f) + if err != nil { + folderLogger.Debug(err) + return err + } + folderLogger.Info("Destroyed") } - folderLogger.Info("Destroyed") } return nil @@ -92,12 +109,14 @@ func (o *ClusterUninstaller) deleteStoragePolicy(ctx context.Context) error { policyName := fmt.Sprintf("openshift-storage-policy-%s", o.InfraID) policyLogger := o.Logger.WithField("StoragePolicy", policyName) policyLogger.Debug("Delete") - err := o.client.DeleteStoragePolicy(ctx, policyName) - if err != nil { - policyLogger.Debug(err) - return err + for _, client := range o.clients { + err := client.DeleteStoragePolicy(ctx, policyName) + if err != nil { + policyLogger.Debug(err) + return err + } + policyLogger.Info("Destroyed") } - policyLogger.Info("Destroyed") return nil } @@ -108,12 +127,14 @@ func (o *ClusterUninstaller) deleteTag(ctx context.Context) error { tagLogger := o.Logger.WithField("Tag", o.InfraID) tagLogger.Debug("Delete") - err := o.client.DeleteTag(ctx, o.InfraID) - if err != nil { - tagLogger.Debug(err) - return err + for _, client := range o.clients { + err := client.DeleteTag(ctx, o.InfraID) + if err != nil { + tagLogger.Debug(err) + return err + } + tagLogger.Info("Deleted") } - tagLogger.Info("Deleted") return nil } @@ -125,19 +146,21 @@ func (o *ClusterUninstaller) deleteTagCategory(ctx context.Context) error { categoryID := "openshift-" + o.InfraID tcLogger := o.Logger.WithField("TagCategory", categoryID) tcLogger.Debug("Delete") - err := o.client.DeleteTagCategory(ctx, categoryID) - if err != nil { - tcLogger.Errorln(err) - return err + for _, client := range o.clients { + err := client.DeleteTagCategory(ctx, categoryID) + if err != nil { + tcLogger.Errorln(err) + return err + } + tcLogger.Info("Deleted") } - tcLogger.Info("Deleted") return nil } -func (o *ClusterUninstaller) stopVirtualMachine(ctx context.Context, vmMO mo.VirtualMachine) error { +func (o *ClusterUninstaller) stopVirtualMachine(ctx context.Context, vmMO mo.VirtualMachine, client API) error { virtualMachineLogger := o.Logger.WithField("VirtualMachine", vmMO.Name) - err := o.client.StopVirtualMachine(ctx, vmMO) + err := client.StopVirtualMachine(ctx, vmMO) if err != nil { virtualMachineLogger.Debug(err) return err @@ -152,17 +175,19 @@ func (o *ClusterUninstaller) stopVirtualMachines(ctx context.Context) error { defer cancel() o.Logger.Debug("Power Off Virtual Machines") - found, err := o.client.ListVirtualMachines(ctx, o.InfraID) - if err != nil { - o.Logger.Debug(err) - return err - } - var errs []error - for _, vmMO := range found { - if !isPoweredOff(vmMO) { - if err := o.stopVirtualMachine(ctx, vmMO); err != nil { - errs = append(errs, err) + for _, client := range o.clients { + found, err := client.ListVirtualMachines(ctx, o.InfraID) + if err != nil { + o.Logger.Debug(err) + return err + } + + for _, vmMO := range found { + if !isPoweredOff(vmMO) { + if err := o.stopVirtualMachine(ctx, vmMO, client); err != nil { + errs = append(errs, err) + } } } } @@ -170,9 +195,9 @@ func (o *ClusterUninstaller) stopVirtualMachines(ctx context.Context) error { return utilerrors.NewAggregate(errs) } -func (o *ClusterUninstaller) deleteVirtualMachine(ctx context.Context, vmMO mo.VirtualMachine) error { +func (o *ClusterUninstaller) deleteVirtualMachine(ctx context.Context, vmMO mo.VirtualMachine, client API) error { virtualMachineLogger := o.Logger.WithField("VirtualMachine", vmMO.Name) - err := o.client.DeleteVirtualMachine(ctx, vmMO) + err := client.DeleteVirtualMachine(ctx, vmMO) if err != nil { virtualMachineLogger.Debug(err) return err @@ -187,16 +212,18 @@ func (o *ClusterUninstaller) deleteVirtualMachines(ctx context.Context) error { defer cancel() o.Logger.Debug("Delete Virtual Machines") - found, err := o.client.ListVirtualMachines(ctx, o.InfraID) - if err != nil { - o.Logger.Debug(err) - return err - } - var errs []error - for _, vmMO := range found { - if err := o.deleteVirtualMachine(ctx, vmMO); err != nil { - errs = append(errs, err) + for _, client := range o.clients { + found, err := client.ListVirtualMachines(ctx, o.InfraID) + if err != nil { + o.Logger.Debug(err) + return err + } + + for _, vmMO := range found { + if err := o.deleteVirtualMachine(ctx, vmMO, client); err != nil { + errs = append(errs, err) + } } } @@ -238,7 +265,9 @@ func (o *ClusterUninstaller) destroyCluster(ctx context.Context) (bool, error) { // Run is the entrypoint to start the uninstall process. func (o *ClusterUninstaller) Run() (*installertypes.ClusterQuota, error) { - defer o.client.Logout() + for _, client := range o.clients { + defer client.Logout() + } err := wait.PollUntilContextCancel( context.Background(), diff --git a/pkg/destroy/vsphere/vsphere_test.go b/pkg/destroy/vsphere/vsphere_test.go index ddb4645481f..168518a0682 100644 --- a/pkg/destroy/vsphere/vsphere_test.go +++ b/pkg/destroy/vsphere/vsphere_test.go @@ -196,7 +196,7 @@ func TestVsphereDeleteFolder(t *testing.T) { for _, edit := range tc.editFuncs { edit(&editedMetadata) } - uninstaller := newWithClient(nullLogger, &editedMetadata, vsphereClient) + uninstaller := newWithClient(nullLogger, &editedMetadata, []API{vsphereClient}) assert.NotNil(t, uninstaller) err := uninstaller.deleteFolder(context.TODO()) if tc.errorMsg != "" { @@ -323,7 +323,7 @@ func TestVsphereStopVirtualMachines(t *testing.T) { for _, edit := range tc.editFuncs { edit(&editedMetadata) } - uninstaller := newWithClient(nullLogger, &editedMetadata, vsphereClient) + uninstaller := newWithClient(nullLogger, &editedMetadata, []API{vsphereClient}) assert.NotNil(t, uninstaller) err := uninstaller.stopVirtualMachines(context.TODO()) if tc.errorMsg != "" { @@ -419,7 +419,7 @@ func TestVsphereDeleteVirtualMachines(t *testing.T) { for _, edit := range tc.editFuncs { edit(&editedMetadata) } - uninstaller := newWithClient(nullLogger, &editedMetadata, vsphereClient) + uninstaller := newWithClient(nullLogger, &editedMetadata, []API{vsphereClient}) assert.NotNil(t, uninstaller) err := uninstaller.deleteVirtualMachines(context.TODO()) if tc.errorMsg != "" { @@ -471,7 +471,7 @@ func TestDeleteStoragePolicy(t *testing.T) { for _, edit := range tc.editFuncs { edit(&editedMetadata) } - uninstaller := newWithClient(nullLogger, &editedMetadata, vsphereClient) + uninstaller := newWithClient(nullLogger, &editedMetadata, []API{vsphereClient}) assert.NotNil(t, uninstaller) err := uninstaller.deleteStoragePolicy(context.TODO()) if tc.errorMsg != "" { @@ -521,7 +521,7 @@ func TestDeleteTag(t *testing.T) { for _, edit := range tc.editFuncs { edit(&editedMetadata) } - uninstaller := newWithClient(nullLogger, &editedMetadata, vsphereClient) + uninstaller := newWithClient(nullLogger, &editedMetadata, []API{vsphereClient}) assert.NotNil(t, uninstaller) err := uninstaller.deleteTag(context.TODO()) if tc.errorMsg != "" { @@ -573,7 +573,7 @@ func TestDeleteTagCategory(t *testing.T) { for _, edit := range tc.editFuncs { edit(&editedMetadata) } - uninstaller := newWithClient(nullLogger, &editedMetadata, vsphereClient) + uninstaller := newWithClient(nullLogger, &editedMetadata, []API{vsphereClient}) assert.NotNil(t, uninstaller) err := uninstaller.deleteTagCategory(context.TODO()) if tc.errorMsg != "" { diff --git a/pkg/infrastructure/clusterapi/clusterapi.go b/pkg/infrastructure/clusterapi/clusterapi.go index c796080784c..4abcd1dd6b3 100644 --- a/pkg/infrastructure/clusterapi/clusterapi.go +++ b/pkg/infrastructure/clusterapi/clusterapi.go @@ -99,10 +99,17 @@ func (i *InfraProvider) Provision(ctx context.Context, dir string, parents asset tfvarsAsset, ) + var clusterIDs []string + // Collect cluster and non-machine-related infra manifests // to be applied during the initial stage. infraManifests := []client.Object{} for _, m := range capiManifestsAsset.RuntimeFiles() { + // Check for cluster definition so that we can collect the names. + if cluster, ok := m.Object.(*clusterv1.Cluster); ok { + clusterIDs = append(clusterIDs, cluster.GetName()) + } + infraManifests = append(infraManifests, m.Object) } @@ -156,6 +163,7 @@ func (i *InfraProvider) Provision(ctx context.Context, dir string, parents asset i.appliedManifests = []client.Object{} // Create the infra manifests. + logrus.Info("Creating infra manifests...") for _, m := range infraManifests { m.SetNamespace(capiutils.Namespace) if err := cl.Create(ctx, m); err != nil { @@ -164,10 +172,13 @@ func (i *InfraProvider) Provision(ctx context.Context, dir string, parents asset i.appliedManifests = append(i.appliedManifests, m) logrus.Infof("Created manifest %+T, namespace=%s name=%s", m, m.GetNamespace(), m.GetName()) } + logrus.Info("Done creating infra manifests") + // Pass cluster kubeconfig and store it in; this is usually the role of a bootstrap provider. - { + for _, capiClusterID := range clusterIDs { + logrus.Infof("Creating kubeconfig entry for capi cluster %v", capiClusterID) key := client.ObjectKey{ - Name: clusterID.InfraID, + Name: capiClusterID, Namespace: capiutils.Namespace, } cluster := &clusterv1.Cluster{} @@ -192,17 +203,28 @@ func (i *InfraProvider) Provision(ctx context.Context, dir string, parents asset if err := wait.PollUntilContextTimeout(ctx, 15*time.Second, timeout, true, func(ctx context.Context) (bool, error) { c := &clusterv1.Cluster{} - if err := cl.Get(ctx, client.ObjectKey{ - Name: clusterID.InfraID, - Namespace: capiutils.Namespace, - }, c); err != nil { - if apierrors.IsNotFound(err) { + var clusters []*clusterv1.Cluster + for _, curClusterID := range clusterIDs { + if err := cl.Get(ctx, client.ObjectKey{ + Name: curClusterID, + Namespace: capiutils.Namespace, + }, c); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + clusters = append(clusters, c) + } + + for _, curCluster := range clusters { + if !curCluster.Status.InfrastructureReady { return false, nil } - return false, err } - cluster = c - return cluster.Status.InfrastructureReady, nil + + cluster = clusters[0] + return true, nil }); err != nil { if wait.Interrupted(err) { return fileList, fmt.Errorf("infrastructure was not ready within %v: %w", timeout, err) diff --git a/pkg/types/validation/featuregate_test.go b/pkg/types/validation/featuregate_test.go index ef4091c9833..61ef384d9cb 100644 --- a/pkg/types/validation/featuregate_test.go +++ b/pkg/types/validation/featuregate_test.go @@ -100,6 +100,49 @@ func TestFeatureGates(t *testing.T) { }(), expected: `^platform.vsphere.hosts: Forbidden: this field is protected by the VSphereStaticIPs feature gate which must be enabled through either the TechPreviewNoUpgrade or CustomNoUpgrade feature set$`, }, + { + name: "vSphere one vcenter is allowed with default Feature Gates", + installConfig: func() *types.InstallConfig { + c := validInstallConfig() + c.FeatureSet = v1.Default + c.VSphere = validVSpherePlatform() + c.VSphere.Hosts = []*vsphere.Host{{Role: "test"}} + return c + }(), + }, + { + name: "vSphere two vcenters is not allowed with Feature Gates disabled", + installConfig: func() *types.InstallConfig { + c := validInstallConfig() + c.FeatureSet = v1.CustomNoUpgrade + c.FeatureGates = []string{"VSphereMultiVCenters=false"} + c.VSphere = validVSpherePlatform() + c.VSphere.VCenters = append(c.VSphere.VCenters, vsphere.VCenter{Server: "additional-vcenter"}) + return c + }(), + expected: `^platform.vsphere.vcenters: Forbidden: this field is protected by the VSphereMultiVCenters feature gate which must be enabled through either the TechPreviewNoUpgrade or CustomNoUpgrade feature set`, + }, + { + name: "vSphere two vcenters is allowed with custom Feature Gate enabled", + installConfig: func() *types.InstallConfig { + c := validInstallConfig() + c.FeatureSet = v1.CustomNoUpgrade + c.FeatureGates = []string{"VSphereMultiVCenters=true"} + c.VSphere = validVSpherePlatform() + c.VSphere.VCenters = append(c.VSphere.VCenters, vsphere.VCenter{Server: "additional-vcenter"}) + return c + }(), + }, + { + name: "vSphere two vcenters is allowed with TechPreview Feature Set", + installConfig: func() *types.InstallConfig { + c := validInstallConfig() + c.FeatureSet = v1.TechPreviewNoUpgrade + c.VSphere = validVSpherePlatform() + c.VSphere.VCenters = append(c.VSphere.VCenters, vsphere.VCenter{Server: "Number2"}) + return c + }(), + }, } for _, tc := range cases { diff --git a/pkg/types/validation/installconfig_test.go b/pkg/types/validation/installconfig_test.go index 3884603b6e8..3c3a9016302 100644 --- a/pkg/types/validation/installconfig_test.go +++ b/pkg/types/validation/installconfig_test.go @@ -742,7 +742,7 @@ func TestValidateInstallConfig(t *testing.T) { c.Platform.VSphere.VCenters[0].Server = "" return c }(), - expectedError: `platform\.vsphere\.vcenters\.server: Required value: must be the domain name or IP address of the vCenter(.*)`, + expectedError: `platform\.vsphere\.vcenters\[0]\.server: Required value: must be the domain name or IP address of the vCenter(.*)`, }, { name: "invalid vsphere folder", diff --git a/pkg/types/vsphere/metadata.go b/pkg/types/vsphere/metadata.go index d06f5ea82f2..703bc641a2b 100644 --- a/pkg/types/vsphere/metadata.go +++ b/pkg/types/vsphere/metadata.go @@ -2,12 +2,24 @@ package vsphere // Metadata contains vSphere metadata (e.g. for uninstalling the cluster). type Metadata struct { + // VCenter is the domain name or IP address of the vCenter. + VCenter string `json:"vCenter,omitempty"` + // Username is the name of the user to use to connect to the vCenter. + Username string `json:"username,omitempty"` + // Password is the password for the user to use to connect to the vCenter. + Password string `json:"password,omitempty"` + // TerraformPlatform is the type... + TerraformPlatform string `json:"terraform_platform"` + // VCenters collection of vcenters when multi vcenter support is enabled + VCenters []VCenters +} + +// VCenters contains information on individual vcenter. +type VCenters struct { // VCenter is the domain name or IP address of the vCenter. VCenter string `json:"vCenter"` // Username is the name of the user to use to connect to the vCenter. Username string `json:"username"` // Password is the password for the user to use to connect to the vCenter. Password string `json:"password"` - // TerraformPlatform is the type... - TerraformPlatform string `json:"terraform_platform"` } diff --git a/pkg/types/vsphere/platform.go b/pkg/types/vsphere/platform.go index 8d9af1c228a..6901d160392 100644 --- a/pkg/types/vsphere/platform.go +++ b/pkg/types/vsphere/platform.go @@ -121,7 +121,7 @@ type Platform struct { // VCenters holds the connection details for services to communicate with vCenter. // Currently only a single vCenter is supported. // +kubebuilder:validation:Optional - // +kubebuilder:validation:MaxItems=1 + // +kubebuilder:validation:MaxItems=3 // +kubebuilder:validation:MinItems=1 VCenters []VCenter `json:"vcenters,omitempty"` // FailureDomains holds the VSpherePlatformFailureDomainSpec which contains diff --git a/pkg/types/vsphere/validation/featuregates.go b/pkg/types/vsphere/validation/featuregates.go index a2776add877..579d84d2d9b 100644 --- a/pkg/types/vsphere/validation/featuregates.go +++ b/pkg/types/vsphere/validation/featuregates.go @@ -8,7 +8,7 @@ import ( "github.com/openshift/installer/pkg/types/featuregates" ) -// GatedFeatures determines all of the OpenStack install config fields that should +// GatedFeatures determines all of the vSphere install config fields that should // be validated to ensure that the proper featuregate is enabled when the field is used. func GatedFeatures(c *types.InstallConfig) []featuregates.GatedInstallConfigFeature { v := c.VSphere @@ -18,5 +18,10 @@ func GatedFeatures(c *types.InstallConfig) []featuregates.GatedInstallConfigFeat Condition: len(v.Hosts) > 0, Field: field.NewPath("platform", "vsphere", "hosts"), }, + { + FeatureGateName: features.FeatureGateVSphereMultiVCenters, + Condition: len(v.VCenters) > 1, + Field: field.NewPath("platform", "vsphere", "vcenters"), + }, } } diff --git a/pkg/types/vsphere/validation/platform.go b/pkg/types/vsphere/validation/platform.go index 6a8a57001bb..89b38284422 100644 --- a/pkg/types/vsphere/validation/platform.go +++ b/pkg/types/vsphere/validation/platform.go @@ -84,26 +84,23 @@ func ValidatePlatform(p *vsphere.Platform, agentBasedInstallation bool, fldPath func validateVCenters(p *vsphere.Platform, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} - if len(p.VCenters) > 1 { - return field.ErrorList{field.TooMany(fldPath, len(p.VCenters), 1)} - } - for _, vCenter := range p.VCenters { + for index, vCenter := range p.VCenters { if len(vCenter.Server) == 0 { - allErrs = append(allErrs, field.Required(fldPath.Child("server"), "must be the domain name or IP address of the vCenter")) + allErrs = append(allErrs, field.Required(fldPath.Index(index).Child("server"), "must be the domain name or IP address of the vCenter")) } else { if err := validate.Host(vCenter.Server); err != nil { - allErrs = append(allErrs, field.Invalid(fldPath.Child("server"), vCenter.Server, "must be the domain name or IP address of the vCenter")) + allErrs = append(allErrs, field.Invalid(fldPath.Index(index).Child("server"), vCenter.Server, "must be the domain name or IP address of the vCenter")) } } if len(vCenter.Username) == 0 { - allErrs = append(allErrs, field.Required(fldPath.Child("username"), "must specify the username")) + allErrs = append(allErrs, field.Required(fldPath.Index(index).Child("username"), "must specify the username")) } if len(vCenter.Password) == 0 { - allErrs = append(allErrs, field.Required(fldPath.Child("password"), "must specify the password")) + allErrs = append(allErrs, field.Required(fldPath.Index(index).Child("password"), "must specify the password")) } if len(vCenter.Datacenters) == 0 { - allErrs = append(allErrs, field.Required(fldPath.Child("datacenters"), "must specify at least one datacenter")) + allErrs = append(allErrs, field.Required(fldPath.Index(index).Child("datacenters"), "must specify at least one datacenter")) } } return allErrs diff --git a/pkg/types/vsphere/validation/platform_test.go b/pkg/types/vsphere/validation/platform_test.go index 238e9f6d097..c8ab5f0efb4 100644 --- a/pkg/types/vsphere/validation/platform_test.go +++ b/pkg/types/vsphere/validation/platform_test.go @@ -238,18 +238,7 @@ func TestValidatePlatform(t *testing.T) { p.VCenters[0].Server = "" return p }(), - expectedError: `test-path\.vcenters\.server: Required value: must be the domain name or IP address of the vCenter(.*)`, - }, - { - name: "Multi-zone platform more than one vCenter", - platform: func() *vsphere.Platform { - p := validPlatform() - p.VCenters = append(p.VCenters, vsphere.VCenter{ - Server: "additional-vcenter", - }) - return p - }(), - expectedError: `^test-path\.vcenters: Too many: 2: must have at most 1 items`, + expectedError: `test-path\.vcenters\[0]\.server: Required value: must be the domain name or IP address of the vCenter(.*)`, }, { name: "Multi-zone platform Capital letters in vCenter", @@ -258,7 +247,7 @@ func TestValidatePlatform(t *testing.T) { p.VCenters[0].Server = "tEsT-vCenter" return p }(), - expectedError: `(.*)test-path\.vcenters.server: Invalid value: "tEsT-vCenter": must be the domain name or IP address of the vCenter`, + expectedError: `(.*)test-path\.vcenters\[0].server: Invalid value: "tEsT-vCenter": must be the domain name or IP address of the vCenter`, }, { name: "Multi-zone missing username", @@ -267,7 +256,7 @@ func TestValidatePlatform(t *testing.T) { p.VCenters[0].Username = "" return p }(), - expectedError: `^test-path\.vcenters.username: Required value: must specify the username$`, + expectedError: `^test-path\.vcenters\[0].username: Required value: must specify the username$`, }, { name: "Multi-zone missing password", @@ -276,7 +265,7 @@ func TestValidatePlatform(t *testing.T) { p.VCenters[0].Password = "" return p }(), - expectedError: `^test-path\.vcenters.password: Required value: must specify the password$`, + expectedError: `^test-path\.vcenters\[0].password: Required value: must specify the password$`, }, { name: "Multi-zone missing datacenter", @@ -285,7 +274,7 @@ func TestValidatePlatform(t *testing.T) { p.VCenters[0].Datacenters = []string{} return p }(), - expectedError: `^test-path\.vcenters.datacenters: Required value: must specify at least one datacenter$`, + expectedError: `^test-path\.vcenters\[0].datacenters: Required value: must specify at least one datacenter$`, }, { name: "Multi-zone platform wrong vCenter name in failureDomain zone",