IPI for vSphere w/ existing resource pool

data/data/install.openshift.io_installconfigs.yaml

@@ -2160,6 +2160,11 @@ spec:
                     description: Password is the password for the user to use to connect
                       to the vCenter.
                     type: string
+                  resourcePool:
+                    description: ResourcePool is the absolute path of the resource
+                      pool where virtual machines will be created. The absolute path
+                      is of the form /<datacenter>/host/<cluster>/Resources/<resourcepool>.
+                    type: string
                   username:
                     description: Username is the name of the user to use to connect
                       to the vCenter.

data/data/vsphere/pre-bootstrap/main.tf

@@ -26,6 +26,10 @@ data "vsphere_compute_cluster" "cluster" {
   datacenter_id = data.vsphere_datacenter.datacenter.id
 }
 
+data "vsphere_resource_pool" "resource_pool" {
+  name = var.vsphere_resource_pool
+}
+
 data "vsphere_datastore" "datastore" {
   name          = var.vsphere_datastore
   datacenter_id = data.vsphere_datacenter.datacenter.id
@@ -42,15 +46,16 @@ data "vsphere_virtual_machine" "template" {
 }
 
 resource "vsphereprivate_import_ova" "import" {
-  name       = var.vsphere_template
-  filename   = var.vsphere_ova_filepath
-  cluster    = var.vsphere_cluster
-  datacenter = var.vsphere_datacenter
-  datastore  = var.vsphere_datastore
-  network    = var.vsphere_network
-  folder     = local.folder
-  tag        = vsphere_tag.tag.id
-  disk_type  = var.vsphere_disk_type
+  name          = var.vsphere_template
+  filename      = var.vsphere_ova_filepath
+  cluster       = var.vsphere_cluster
+  resource_pool = var.vsphere_resource_pool
+  datacenter    = var.vsphere_datacenter
+  datastore     = var.vsphere_datastore
+  network       = var.vsphere_network
+  folder        = local.folder
+  tag           = vsphere_tag.tag.id
+  disk_type     = var.vsphere_disk_type
 }
 
 resource "vsphere_tag_category" "category" {

data/data/vsphere/pre-bootstrap/outputs.tf

@@ -1,5 +1,5 @@
 output "resource_pool" {
-  value = data.vsphere_compute_cluster.cluster.resource_pool_id
+  value = data.vsphere_resource_pool.resource_pool.id
 }
 
 output "datastore" {

data/data/vsphere/variables-vsphere.tf

@@ -22,6 +22,11 @@ variable "vsphere_cluster" {
   description = "This is the name of the vSphere cluster."
 }
 
+variable "vsphere_resource_pool" {
+  type        = string
+  description = "This is the absolute path to the vSphere resource pool."
+}
+
 variable "vsphere_datacenter" {
   type        = string
   description = "This is the name of the vSphere data center."

docs/user/vsphere/customization.md

@@ -10,6 +10,7 @@ Beyond the [platform-agnostic `install-config.yaml` properties](../customization
 * `datacenter` (required string): The name of the datacenter to use in the vCenter.
 * `defaultDatastore` (required string): The default datastore to use for provisioning volumes.
 * `folder` (optional string): The absolute path of an existing folder where the installer should create VMs. The absolute path is of the form `/example_datacenter/vm/example_folder/example_subfolder`. If a value is specified, the folder must exist. If no value is specified, a folder named with the cluster ID will be created in the `datacenter` VM folder.
+* `resourcePool` (optional string): The absolute path of an existing resource pool where the installer should create VMs. The absolute path is of the form `/example_datacenter/host/example_cluster/Resources/example_resource_pool/optionally_sub_resource_pool`. If a value is specified, the resource pool must exist. If no value is specified, resources will be installed in the root of the cluster `/example_datacenter/host/example_cluster/Resources`.
 
 ## Machine pools
 

docs/user/vsphere/privileges.md

@@ -7,14 +7,14 @@ If the provided user has global admin privileges, no further action for permissi
 
 The tables below describe the absolute minimal set of privileges to install and run OpenShift including Machine management and the vSphere Storage provider.
 
-### Fundamental Privileges
+### Fundamental (minimum) Privileges
 
-These privileges are necessary for OpenShift clusters on vSphere and are sufficient to install into an existing virtual machine folder. The privileges in the next section are necessary for the installer to provision a folder, which is the default behavior if no folder is specified in the install config.
+These privileges are necessary for OpenShift clusters on vSphere and are sufficient to install into an existing virtual machine folder and an existing resource pool. The privileges in the next section are necessary for the installer to provision a folder, which is the default behavior if no folder is specified in the install config. The priviliges in the third section are necessary for the installer to create VMs in the root of the cluster, which is the default behavior if no resource pool is specified in the install config.
 
 Role Name | vSphere object | Privilege Set
 --- | --- | ---
 openshift-vcenter-level | vSphere vCenter | Cns.Searchable<br/>InventoryService.Tagging.AttachTag<br/>InventoryService.Tagging.CreateCategory<br/>InventoryService.Tagging.CreateTag<br/>InventoryService.Tagging.DeleteCategory<br/>InventoryService.Tagging.DeleteTag<br/>InventoryService.Tagging.EditCategory<br/>InventoryService.Tagging.EditTag<br/>Sessions.ValidateSession<br/>StorageProfile.View
-openshift-cluster-level | vSphere vCenter Cluster | Host.Config.Storage<br/>Resource.AssignVMToPool<br/>VApp.AssignResourcePool<br/>VApp.Import<br/>VirtualMachine.Config.AddNewDisk
+openshift-resourcepool-level | vSphere vCenter Resource Pool | Host.Config.Storage<br/>Resource.AssignVMToPool<br/>VApp.AssignResourcePool<br/>VApp.Import<br/>VirtualMachine.Config.AddNewDisk
 openshift-datastore-level| vSphere Datastore | Datastore.AllocateSpace<br/>Datastore.Browse<br/>Datastore.FileManagement
 openshift-portgroup-level | vSphere Port Group | Network.Assign
 openshift-folder-level| Virtual Machine Folder | Resource.AssignVMToPool<br/>VApp.Import<br/>VirtualMachine.Config.AddExistingDisk<br/>VirtualMachine.Config.AddNewDisk<br/>VirtualMachine.Config.AddRemoveDevice<br/>VirtualMachine.Config.AdvancedConfig<br/>VirtualMachine.Config.Annotation<br/>VirtualMachine.Config.CPUCount<br/>VirtualMachine.Config.DiskExtend<br/>VirtualMachine.Config.DiskLease<br/>VirtualMachine.Config.EditDevice<br/>VirtualMachine.Config.Memory<br/>VirtualMachine.Config.RemoveDisk<br/>VirtualMachine.Config.Rename<br/>VirtualMachine.Config.ResetGuestInfo<br/>VirtualMachine.Config.Resource<br/>VirtualMachine.Config.Settings<br/>VirtualMachine.Config.UpgradeVirtualHardware<br/>VirtualMachine.Interact.GuestControl<br/>VirtualMachine.Interact.PowerOff<br/>VirtualMachine.Interact.PowerOn<br/>VirtualMachine.Interact.Reset<br/>VirtualMachine.Inventory.Create<br/>VirtualMachine.Inventory.CreateFromExisting<br/>VirtualMachine.Inventory.Delete<br/>VirtualMachine.Provisioning.Clone
@@ -29,13 +29,35 @@ Role Name | vSphere object | Privilege Set
 --- | --- | ---
 openshift-datacenter-level| vSphere vCenter Datacenter | Resource.AssignVMToPool<br/>VApp.Import<br/>VirtualMachine.Config.AddExistingDisk<br/>VirtualMachine.Config.AddNewDisk<br/>VirtualMachine.Config.AddRemoveDevice<br/>VirtualMachine.Config.AdvancedConfig<br/>VirtualMachine.Config.Annotation<br/>VirtualMachine.Config.CPUCount<br/>VirtualMachine.Config.DiskExtend<br/>VirtualMachine.Config.DiskLease<br/>VirtualMachine.Config.EditDevice<br/>VirtualMachine.Config.Memory<br/>VirtualMachine.Config.RemoveDisk<br/>VirtualMachine.Config.Rename<br/>VirtualMachine.Config.ResetGuestInfo<br/>VirtualMachine.Config.Resource<br/>VirtualMachine.Config.Settings<br/>VirtualMachine.Config.UpgradeVirtualHardware<br/>VirtualMachine.Interact.GuestControl<br/>VirtualMachine.Interact.PowerOff<br/>VirtualMachine.Interact.PowerOn<br/>VirtualMachine.Interact.Reset<br/>VirtualMachine.Inventory.Create<br/>VirtualMachine.Inventory.CreateFromExisting<br/>VirtualMachine.Inventory.Delete<br/>VirtualMachine.Provisioning.Clone<br/>Folder.Create<br/>Folder.Delete
 
+### Resources installed in root of cluster (no resource pool)
+
+Including the role-set above one additional role needs to be created if the installer is to create VMs in the root of the cluster. Note that the privileges applied at the cluster-level in this case are the same as those applied at the resource-pool-level above.
+
+Role Name | vSphere object | Privilege Set
+--- | --- | ---
+openshift-cluster-level | vSphere vCenter Cluster | Host.Config.Storage<br/>Resource.AssignVMToPool<br/>VApp.AssignResourcePool<br/>VApp.Import<br/>VirtualMachine.Config.AddNewDisk
+
 ## Permission assignments
 
 The easiest way to ensure proper permissions is to grant Global Permissions to the user with the privileges above. Otherwise, it is necessary to ensure that the user with the listed privileges has permissions granted on all necessary entities in the vCenter.
 
 For more information, consult [vSphere Permissions and User Management Tasks][vsphere-perms]
 
-### Precreated virtual machine folder
+### Precreated virtual machine folder and resource pool
+
+Role Name | Propagate | Entity
+--- | --- | ---
+openshift-vcenter-level | False | vSphere vCenter
+ReadOnly | False | vSphere vCenter Datacenter
+ReadOnly | True | vSphere vCenter Cluster
+openshift-resourcepool-level | True | vSphere vCenter Resource Pool
+openshift-datastore-level | False | vSphere vCenter Datastore
+ReadOnly | False | vSphere Switch
+openshift-portgroup-level | False | vSphere Port Group
+openshift-folder-level | True | vSphere vCenter Virtual Machine folder
+
+
+### Precreated virtual machine folder without resource pool
 
 Role Name | Propagate | Entity
 --- | --- | ---
@@ -48,7 +70,7 @@ openshift-portgroup-level | False | vSphere Port Group
 openshift-folder-level | True | vSphere vCenter Virtual Machine folder
 
 
-### Installer created virtual machine folder
+### Installer created virtual machine folder without resource pool
 Role Name | Propagate | Entity
 --- | --- | ---
 openshift-vcenter-level | False | vSphere vCenter

pkg/asset/installconfig/vsphere/client.go

@@ -21,6 +21,7 @@ type Finder interface {
 	Folder(ctx context.Context, path string) (*object.Folder, error)
 	NetworkList(ctx context.Context, path string) ([]object.NetworkReference, error)
 	Network(ctx context.Context, path string) (object.NetworkReference, error)
+	ResourcePool(ctx context.Context, path string) (*object.ResourcePool, error)
 }
 
 // NewFinder creates a new client that conforms with the Finder interface and returns a

pkg/asset/installconfig/vsphere/validation.go

@@ -73,6 +73,7 @@ func validateProvisioning(finder Finder, ic *types.InstallConfig) error {
 	allErrs := field.ErrorList{}
 	allErrs = append(allErrs, validation.ValidateForProvisioning(ic.Platform.VSphere, field.NewPath("platform").Child("vsphere"))...)
 	allErrs = append(allErrs, folderExists(finder, ic, field.NewPath("platform").Child("vsphere").Child("folder"))...)
+	allErrs = append(allErrs, resourcePoolExists(finder, ic, field.NewPath("platform").Child("vsphere").Child("resourcePool"))...)
 
 	return allErrs.ToAggregate()
 }
@@ -115,3 +116,22 @@ func validateNetwork(finder Finder, p *vsphere.Platform, fldPath *field.Path) fi
 	}
 	return nil
 }
+
+// resourcePoolExists returns an error if a resourcePool is specified in the vSphere platform but a resourcePool with that name is not found in the datacenter.
+func resourcePoolExists(finder Finder, ic *types.InstallConfig, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+	cfg := ic.VSphere
+
+	// If no resourcePool is specified, skip this check as the root resourcePool will be used.
+	if cfg.ResourcePool == "" {
+		return allErrs
+	}
+
+	ctx, cancel := context.WithTimeout(context.TODO(), 60*time.Second)
+	defer cancel()
+
+	if _, err := finder.ResourcePool(ctx, cfg.ResourcePool); err != nil {
+		return append(allErrs, field.Invalid(fldPath, cfg.ResourcePool, err.Error()))
+	}
+	return nil
+}

pkg/asset/machines/vsphere/machines.go

@@ -69,6 +69,9 @@ func provider(clusterID string, platform *vsphere.Platform, mpool *vsphere.Machi
 	if platform.Folder != "" {
 		folder = platform.Folder
 	}
+	if platform.ResourcePool != "" {
+		resourcePool = platform.ResourcePool
+	}
 
 	return &vsphereapis.VSphereMachineProviderSpec{
 		TypeMeta: metav1.TypeMeta{

pkg/asset/manifests/vsphere/cloudproviderconfig.go

@@ -30,6 +30,7 @@ func CloudProviderConfig(folderPath string, p *vspheretypes.Platform) (string, e
 	printIfNotEmpty(buf, "datacenter", p.Datacenter)
 	printIfNotEmpty(buf, "default-datastore", p.DefaultDatastore)
 	printIfNotEmpty(buf, "folder", folderPath)
+	printIfNotEmpty(buf, "resourcepool-path", p.ResourcePool)
 	fmt.Fprintln(buf, "")
 
 	fmt.Fprintf(buf, "[VirtualCenter %q]\n", p.VCenter)

pkg/terraform/exec/plugins/vsphereprivate/resource_vsphereprivate_import_ova.go

@@ -65,6 +65,13 @@ func resourceVSpherePrivateImportOva() *schema.Resource {
 				ForceNew:     true,
 				ValidateFunc: validation.NoZeroValues,
 			},
+			"resource_pool": {
+				Type:         schema.TypeString,
+				Description:  "The absolute path to the resource pool.",
+				Required:     true,
+				ForceNew:     true,
+				ValidateFunc: validation.NoZeroValues,
+			},
 			"network": {
 				Type:         schema.TypeString,
 				Description:  "The name of a network that the virtual machine will use.",
@@ -114,7 +121,7 @@ type importOvaParams struct {
 	HardwareVersion string
 }
 
-func findImportOvaParams(client *vim25.Client, datacenter, cluster, datastore, network, folder string) (*importOvaParams, error) {
+func findImportOvaParams(client *vim25.Client, datacenter, cluster, resourcePool, datastore, network, folder string) (*importOvaParams, error) {
 	var ccrMo mo.ClusterComputeResource
 
 	ctx, cancel := context.WithTimeout(context.TODO(), defaultAPITimeout)
@@ -140,6 +147,14 @@ func findImportOvaParams(client *vim25.Client, datacenter, cluster, datastore, n
 
 	clusterPath := fmt.Sprintf("/%s/host/%s", datacenter, cluster)
 
+	// Find the resource pool object by using its path provided by install-config,
+	// or generated in pkg/asset/machines/vsphere/machines.go
+	resourcePoolObj, err := finder.ResourcePool(ctx, resourcePool)
+	if err != nil {
+		return nil, err
+	}
+	importOvaParams.ResourcePool = resourcePoolObj
+
 	// Find the cluster object by the datacenter and cluster name to
 	// generate the path e.g. /datacenter/host/cluster
 	clusterComputeResource, err := finder.ClusterComputeResource(ctx, clusterPath)
@@ -261,12 +276,7 @@ func findImportOvaParams(client *vim25.Client, datacenter, cluster, datastore, n
 		}
 
 		if foundDatastore && foundNetwork {
-			importOvaParams.Host = hostObj
-			resourcePool, err := hostObj.ResourcePool(ctx)
-			if err != nil {
-				return nil, err
-			}
-			importOvaParams.ResourcePool = resourcePool
+			return importOvaParams, nil
 		}
 	}
 
@@ -324,6 +334,7 @@ func resourceVSpherePrivateImportOvaCreate(d *schema.ResourceData, meta interfac
 	importOvaParams, err := findImportOvaParams(client,
 		d.Get("datacenter").(string),
 		d.Get("cluster").(string),
+		d.Get("resource_pool").(string),
 		d.Get("datastore").(string),
 		d.Get("network").(string),
 		d.Get("folder").(string))

pkg/tfvars/vsphere/vsphere.go

@@ -20,6 +20,7 @@ type config struct {
 	NumCPUs           int32            `json:"vsphere_control_plane_num_cpus"`
 	NumCoresPerSocket int32            `json:"vsphere_control_plane_cores_per_socket"`
 	Cluster           string           `json:"vsphere_cluster"`
+	ResourcePool      string           `json:"vsphere_resource_pool"`
 	Datacenter        string           `json:"vsphere_datacenter"`
 	Datastore         string           `json:"vsphere_datastore"`
 	Folder            string           `json:"vsphere_folder"`
@@ -64,6 +65,7 @@ func TFVars(sources TFVarsSources) ([]byte, error) {
 		NumCPUs:           controlPlaneConfig.NumCPUs,
 		NumCoresPerSocket: controlPlaneConfig.NumCoresPerSocket,
 		Cluster:           sources.Cluster,
+		ResourcePool:      controlPlaneConfig.Workspace.ResourcePool,
 		Datacenter:        controlPlaneConfig.Workspace.Datacenter,
 		Datastore:         controlPlaneConfig.Workspace.Datastore,
 		Folder:            folderRelPath,

pkg/types/validation/installconfig_test.go

@@ -728,7 +728,19 @@ func TestValidateInstallConfig(t *testing.T) {
 				c.Platform.VSphere.Folder = "my-folder"
 				return c
 			}(),
-			expectedError: `^platform\.vsphere.folder: Invalid value: \"my-folder\": folder must be absolute path: expected prefix /test-datacenter/vm/$`,
+			expectedError: `^platform\.vsphere\.folder: Invalid value: \"my-folder\": folder must be absolute path: expected prefix /test-datacenter/vm/$`,
+		},
+		{
+			name: "invalid vsphere resource pool",
+			installConfig: func() *types.InstallConfig {
+				c := validInstallConfig()
+				c.Platform = types.Platform{
+					VSphere: validVSpherePlatform(),
+				}
+				c.Platform.VSphere.ResourcePool = "my-resource-pool"
+				return c
+			}(),
+			expectedError: `^platform\.vsphere\.resourcePool: Invalid value: \"my-resource-pool\": resourcePool must be absolute path: expected prefix /test-datacenter/host/<cluster>/Resources/$`,
 		},
 		{
 			name: "empty proxy settings",

pkg/types/vsphere/platform.go

@@ -38,6 +38,10 @@ type Platform struct {
 	// Cluster is the name of the cluster virtual machines will be cloned into.
 	Cluster string `json:"cluster,omitempty"`
 
+	// ResourcePool is the absolute path of the resource pool where virtual machines will be
+	// created. The absolute path is of the form /<datacenter>/host/<cluster>/Resources/<resourcepool>.
+	ResourcePool string `json:"resourcePool,omitempty"`
+
 	// ClusterOSImage overrides the url provided in rhcos.json to download the RHCOS OVA
 	ClusterOSImage string `json:"clusterOSImage,omitempty"`
 

pkg/types/vsphere/validation/platform.go

@@ -45,6 +45,11 @@ func ValidatePlatform(p *vsphere.Platform, fldPath *field.Path) field.ErrorList
 		allErrs = append(allErrs, validateFolder(p, fldPath)...)
 	}
 
+	// resource pool is optional, but if provided should pass validation
+	if len(p.ResourcePool) != 0 {
+		allErrs = append(allErrs, validateResourcePool(p, fldPath)...)
+	}
+
 	return allErrs
 }
 
@@ -87,7 +92,7 @@ func validateVIPs(p *vsphere.Platform, fldPath *field.Path) field.ErrorList {
 	return allErrs
 }
 
-// validateFolder checks that a provided folder is in absolute path in the correct datacenter.
+// validateFolder checks that a provided folder is an absolute path in the correct datacenter.
 func validateFolder(p *vsphere.Platform, fldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}
 
@@ -104,3 +109,25 @@ func validateFolder(p *vsphere.Platform, fldPath *field.Path) field.ErrorList {
 
 	return allErrs
 }
+
+// validateResourcePool checks that a provided resource pool is an absolute path in the correct cluster.
+func validateResourcePool(p *vsphere.Platform, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	dc := p.Datacenter
+	if len(dc) == 0 {
+		dc = "<datacenter>"
+	}
+	cluster := p.Cluster
+	if len(cluster) == 0 {
+		cluster = "<cluster>"
+	}
+	expectedPrefix := fmt.Sprintf("/%s/host/%s/Resources/", dc, cluster)
+
+	if !strings.HasPrefix(p.ResourcePool, expectedPrefix) {
+		errMsg := fmt.Sprintf("resourcePool must be absolute path: expected prefix %s", expectedPrefix)
+		allErrs = append(allErrs, field.Invalid(fldPath.Child("resourcePool"), p.ResourcePool, errMsg))
+	}
+
+	return allErrs
+}