From 97512e04e690fd5865727265f91b5dce27c7198c Mon Sep 17 00:00:00 2001 From: Iacopo Rozzo Date: Fri, 3 Apr 2020 15:48:34 +0200 Subject: [PATCH] Move vSphere cloud provider and RHEL image preparation doc to dedicated file and improve it --- docs/cloud-provider.md | 31 +---- docs/operating-system.md | 4 +- docs/rhel-custom-image.md | 34 ++--- docs/vsphere.md | 266 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 271 insertions(+), 64 deletions(-) diff --git a/docs/cloud-provider.md b/docs/cloud-provider.md index a94fa5643..7069cb568 100644 --- a/docs/cloud-provider.md +++ b/docs/cloud-provider.md @@ -277,33 +277,4 @@ tags: ## vSphere -### machine.spec.providerConfig.cloudProviderSpec -```yaml -# Can also be set via the env var 'VSPHERE_USERNAME' on the machine-controller -username: '<< VSPHERE_USERNAME >>' -# Can also be set via the env var 'VSPHERE_ADDRESS' on the machine-controller -# example: 'https://your-vcenter:8443'. '/sdk' gets appended automatically -vsphereURL: '<< VSPHERE_ADDRESS >>' -# Can also be set via the env var 'VSPHERE_PASSWORD' on the machine-controller -password: "<< VSPHERE_PASSWORD >>" -# datacenter name -datacenter: datacenter1 -# VM template name -templateVMName: ubuntu-template -# Optional. Sets the networks on the VM. If no network is specified, the template default will be used. -vmNetName: network1 -# Optional -folder: folder1 -cluster: cluster1 -# either datastore or datastoreCluster have to be provided. -datastore: datastore1 -datastoreCluster: datastore-cluster1 -# Can also be set via the env var 'VSPHERE_ALLOW_INSECURE' on the machine-controller -allowInsecure: true -# instance resources -cpus: 2 -memoryMB: 2048 -# Optional: Resize the root disk to this size. Must be bigger than the existing size -# Default is to leave the disk at the same size as the template -diskSizeGB: 10 -``` +Refer to the [VSphere](./vsphere.md#provider-configuration) specific documentation. diff --git a/docs/operating-system.md b/docs/operating-system.md index 9ad6f6ece..08c034cbf 100644 --- a/docs/operating-system.md +++ b/docs/operating-system.md @@ -34,8 +34,8 @@ Machine controller may work with other OS versions that are not listed in the ta | | Versions | |---|---| | CentOS | 7.4.x, 7.6.x, 7.7.x | -| CoreOS | 1855.4.0, 2079.3.0, 2345.3.0 | -| RHEL | 8.0 | +| CoreOS | 1855.4.0, 2079.x.x, 2135.x.x, 2191.x.x, 2247.x.x, 2345.x.x | +| RHEL | 8.0, 8.1 | | SLES | SLES 15 SP1 | | Ubuntu | 18.04 LTS | diff --git a/docs/rhel-custom-image.md b/docs/rhel-custom-image.md index 917db26e9..ac5db0ee3 100644 --- a/docs/rhel-custom-image.md +++ b/docs/rhel-custom-image.md @@ -1,6 +1,6 @@ # RedHat Enterprise Linux -Cloud providers which are listed below, support using rhel as an operating system option: +Cloud providers which are listed below, support using RHEL as an operating system option: - AWS - Azure @@ -9,18 +9,19 @@ Cloud providers which are listed below, support using rhel as an operating syste - Openstack - vSphere -#### AWS: -For amazon web service cloud provider, First of all the RHEL gold image AMIs have to be enabled from the -[RedHat Customer Portal](https://access.redhat.com/public-cloud/aws) (this requires a [cloud-provider subscription](https://access.redhat.com/public-cloud)). -.Afterwards, new images will be added to the aws account under EC2-> Images-> AMIs-> Private Images. Once the images are available in the aws account, -the image id for rhel(supported versions are mentioned [here](https://github.com/kubermatic/machine-controller/blob/master/docs/operating-system.md)) should be then added to the `MachineDeployment` spec to the field `ami`. +#### AWS +First of all the RHEL gold image AMIs have to be enabled from the [RedHat Customer Portal](https://access.redhat.com/public-cloud/aws) (this requires a [cloud-provider subscription](https://access.redhat.com/public-cloud)). + +Afterwards, new images will be added to the aws account under EC2-> Images-> AMIs-> Private Images. +Once the images are available in the aws account, the image id for RHEL (supported versions are mentioned [here](./operating-system.md#supported-os-versions)) should be then added to the `MachineDeployment` spec to the field `ami`. #### Azure RedHat provides images for Azure, [documentation](https://access.redhat.com/articles/uploading-rhel-image-to-azure) is available on RH customer portal. + The `MachineDeployment` field `.spec.template.spec.providerSpec.value.cloudProviderSpec.imageID` should reference the ID of the uploaded VM. **Note:** -Azure rhel images starting from 7.6.x don't support cloud-init as their documentation states [here](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/using-cloud-init#rhel). +Azure RHEL images starting from 7.6.x don't support cloud-init as their documentation states [here](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/using-cloud-init#rhel). Thus, custom images can be used with a cloud-init pre-installed to solve this issue. Follow this [documentation](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/cloudinit-prepare-custom-image) to prepare an image with cloud-init support. @@ -28,12 +29,11 @@ to prepare an image with cloud-init support. RedHat also provides Gold Access Image for GCE and those can be fetched just like aws and azure. The `MachineDeployment` field `.spec.template.spec.providerSpec.value.cloudProviderSpec.customImage` should reference the ID of the used image. **Note:** -Same as for Azure, rhel images in GCE don't support cloud-init. Thus, custom images can be used with a cloud-init pre-installed -to solve this issue. Follow this [documentation](https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/8/html/deploying_red_hat_enterprise_linux_8_on_public_cloud_platforms/assembly_deploying-a-rhel-image-as-a-compute-engine-instance-on-google-cloud-platform_deploying-a-virtual-machine-on-aws) to upload custom rhel -images in order to use it for running rhel instances. +RHEL images in GCE don't support cloud-init. Thus, custom images can be used with a cloud-init pre-installed +to solve this issue. Follow this [documentation](https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/8/html/deploying_red_hat_enterprise_linux_8_on_public_cloud_platforms/assembly_deploying-a-rhel-image-as-a-compute-engine-instance-on-google-cloud-platform_deploying-a-virtual-machine-on-aws) to upload custom RHEL images. #### KubeVirt -In order to create machines which run rhel as an operating system in KubeVirt cloud provider, the image should be available and fetched +In order to create machines which run RHEL as an operating system in KubeVirt cloud provider, the image should be available and fetched via an endpoint. This endpoint should be then added to the `MachineDeployment` field `.spec.template.spec.providerSpec.value.cloudProviderSpec.sourceURL`. For more information about the supported images please refer to this documentation from KubeVirt CDI [here](https://kubevirt.io/2018/containerized-data-importer.html) @@ -42,13 +42,5 @@ Once RHEL images(e.g: Red Hat Enterprise Linux 8.x KVM Guest Image) is uploaded the `MachineDeployment` field `.spec.template.spec.providerSpec.value.cloudProviderSpec.image`. #### vSphere -To rhel os for vSphere instance, a template for the rhel machine should be created or a clone from a rhel machine. To upload rhel -image to vSphere, follow these steps to create instances from a cloned machine: - -- Download Red Hat Enterprise Linux 8.x KVM Guest Image from Red Hat Customer Portal. -- The image has the format `qcow2` thus should be converted to `vmdk` by running the command: `qemu-img convert -f qcow2 rhel.qcow2 -O vmdk newRHEL.vmdk` -- Upload the image to vSphere Datastore. Preferably use [`govc`](https://github.com/vmware/govmomi/blob/master/govc/USAGE.md#datastoreupload) -- Once the image is uploaded to ESXi host, run `vmkfstools -i newRHEL.vmdk outputRHEL.vmdk -d thin`to ensure that, the `vmdk` image is ESXi compatible. -- Create a new instance using that image. During the machine creation process, at the `Customize Hardware` step, press on ADD NEW DEVICE and select Existing Hard Disk. -- In the Existing Hard Disk wizard select the rhel image file and then create the instance. -- Use the instance name to clone rhel machine by updating the `MachineDeployment` field `.spec.template.spec.providerSpec.value.cloudProviderSpec.templateVMName`. + +Find [here](./vsphere.md#RHEL) how to deploy a tempate VM in vSphere. diff --git a/docs/vsphere.md b/docs/vsphere.md index e14e6433b..aebc1d1d5 100644 --- a/docs/vsphere.md +++ b/docs/vsphere.md @@ -1,23 +1,267 @@ # VMware vSphere +## Supported versions + +* 6.5 +* 6.7 + +## Template VMs preparation + To use the machine-controller to create machines on VMWare vsphere, you must first -create a template. +create a VM to be used as a template. + +*Note that:* +`template VMs` in this document refers to regular VMs and not +[VM Templates][vm_templates] according to vSphere terminology. +The difference is quite subtle, but VM Templates are not supported yet by +`machine controller`. -Ubuntu & CoreOS: +### Create template VM from OVA -1. Go into the VSphere WebUI, select your datacenter, right click onto it and choose "Deploy OVF Template" -2. Fill in the "URL" field with the appropriate url: - * Ubuntu: `https://cloud-images.ubuntu.com/releases/18.04/release/ubuntu-18.04-server-cloudimg-amd64.ova` - * Container Linux: `https://stable.release.core-os.net/amd64-usr/current/coreos_production_vmware_ova.ova` +To see where to locate the OVAs go to the OS specific section. + +#### WebUI procedure + +1. Go into the vSphere WebUI, select your datacenter, right click onto it and choose "Deploy OVF Template" +2. Fill in the "URL" field with the appropriate url pointing to the `OVA` file 3. Click through the dialog until "Select storage" 4. Select the same storage you want to use for your machines 5. Select the same network you want to use for your machines 6. Leave everyhting in the "Customize Template" and "Ready to complete" dialog as it is 7. Wait until the VM got fully imported and the "Snapshots" => "Create Snapshot" button is not grayed out anymore -CentOS: +#### Command-line procedure + +Prerequisites: + +* [GOVC](https://github.com/vmware/govmomi/tree/master/govc): tested on version 0.22.1 +* [jq](https://stedolan.github.io/jq/) + +Procedure: + +1. Download the `OVA` for the targeted OS. + + ``` + curl -sL "${OVA_URL}" -O . + ``` + +2. Extract the specs from the `OVA`: + + ``` + govc import.spec $(basename "${OVA_URL}") | jq -r > options.json + ``` + +3. Edit the `options.json` file with your text editor of choice. + + * Edit the `NetworkMapping` to point to the correct network. + * Make sure that `PowerOn` is set to `false`. + * Make sure that `MarkAsTemplate` is set to `false`. + * Verify the other properties and customize according to your needs. + e.g. + + ```json + { + "DiskProvisioning": "flat", + "IPAllocationPolicy": "dhcpPolicy", + "IPProtocol": "IPv4", + "PropertyMapping": [ + { + "Key": "guestinfo.hostname", + "Value": "" + }, + { + "Key": "guestinfo.coreos.config.data", + "Value": "" + }, + { + "Key": "guestinfo.coreos.config.url", + "Value": "" + }, + { + "Key": "guestinfo.coreos.config.data.encoding", + "Value": "" + }, + { + "Key": "guestinfo.interface.0.name", + "Value": "" + }, + { + "Key": "guestinfo.interface.0.mac", + "Value": "" + }, + { + "Key": "guestinfo.interface.0.dhcp", + "Value": "no" + }, + { + "Key": "guestinfo.interface.0.role", + "Value": "public" + }, + { + "Key": "guestinfo.interface.0.ip.0.address", + "Value": "" + }, + { + "Key": "guestinfo.interface.0.route.0.gateway", + "Value": "" + }, + { + "Key": "guestinfo.interface.0.route.0.destination", + "Value": "" + }, + { + "Key": "guestinfo.dns.server.0", + "Value": "" + }, + { + "Key": "guestinfo.dns.server.1", + "Value": "" + } + ], + "NetworkMapping": [ + { + "Name": "VM Network", + "Network": "Loodse Default" + } + ], + "MarkAsTemplate": false, + "PowerOn": false, + "InjectOvfEnv": false, + "WaitForIP": false, + "Name": null + } + ``` + +4. Create a VM from the `OVA`: + + ``` + govc import.ova -options=options.json $(basename "${OVA_URL}") + ``` + +### Create template VM from qcow2 + +Prerequisites: + +* vSphere (tested on version 6.7) +* GOVC (tested on version 0.22.1) +* qemu-img (tested on version 4.2.0) +* curl or wget + +Procedure: + +1. Download the guest image in qcow2 format end export an environment variable + whith the name of the file. + + ``` + # The URL below is just an example + image_url="https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2" + image_name="$(basename -- "${image_url}" | sed 's/.qcow2$//g')" + curl -sL "${image_url}" -O . + ``` + +2. Convert it to vmdk e.g. + + ``` + qemu-img convert -O vmdk -o subformat=streamOptimized "./${image_name}.qcow2" "${image_name}.vmdk" + ``` + +3. Upload to vSphere using WebUI or GOVC: + + Make sure to replace the parameters on the command below with the correct + values specific to yout vSphere environment. + + ``` + govc import.vmdk -dc=dc-1 -pool=/dc-1/host/cl-1/Resources -ds=ds-1 "./${image_name}.vmdk" + ``` + +4. Inflate the created disk (see vmware [documentation][inflate_thin_virtual_disks] for more details) + + ``` + govc datastore.disk.inflate -dc dc-1 -ds ds-1 "${image_name}/${image_name}.vmdk" + ``` + +5. Create a new virtual machine using that image with vSphere WebUI. +6. During the `Customize Hardware` step: + 1. Remove the disk present by default + 2. Click on `ADD NEW DEVICE`, select `Existing Hard Disk` and select the + disk previously created. +7. The vm is ready to be used by the `MachineController` by referencing its name in the field `.spec.template.spec.providerSpec.value.cloudProviderSpec.templateVMName` of the `MachineDeployment`. + +### OS images + +Information about supported OS versions can be found [here](./operating-system.md#supported-os-versions). + +#### Ubuntu + +Ubuntu OVA template can be foud at . + +Follow [OVA](#create-template-vm-from-ova) template VM creation guide. + +#### CoreOS + +CoreOS OVA template can be foud at . + +Follow [OVA](#create-template-vm-from-ova) template VM creation guide. + +#### RHEL + +Red Hat Enterprise Linux 8.x KVM Guest Image can be found at [Red Hat Customer Portal][rh_portal_rhel8]. + +Follow [qcow2](#create-template-vm-from-qcow2) template VM creation guide. + +#### CentOS + +CentOS 7 image can be found at the following link: . + +Follow [qcow2](#create-template-vm-from-qcow2) template VM creation guide. + +## Provider configuration + +VSphere provider accepts the following configuration parameters: + +```yaml +# Can also be set via the env var 'VSPHERE_USERNAME' on the machine-controller +username: '<< VSPHERE_USERNAME >>' +# Can also be set via the env var 'VSPHERE_ADDRESS' on the machine-controller +# example: 'https://your-vcenter:8443'. '/sdk' gets appended automatically +vsphereURL: '<< VSPHERE_ADDRESS >>' +# Can also be set via the env var 'VSPHERE_PASSWORD' on the machine-controller +password: "<< VSPHERE_PASSWORD >>" +# datacenter name +datacenter: datacenter1 +# VM template name +templateVMName: ubuntu-template +# Optional. Sets the networks on the VM. If no network is specified, the template default will be used. +vmNetName: network1 +# Optional +folder: folder1 +cluster: cluster1 +# either datastore or datastoreCluster have to be provided. +datastore: datastore1 +datastoreCluster: datastore-cluster1 +# Can also be set via the env var 'VSPHERE_ALLOW_INSECURE' on the machine-controller +allowInsecure: true +# instance resources +cpus: 2 +memoryMB: 2048 +# Optional: Resize the root disk to this size. Must be bigger than the existing size +# Default is to leave the disk at the same size as the template +diskSizeGB: 10 +``` + +### Datastore and DatastoreCluster + +A `Datastore` is the basic unit of storage abstraction in VSphere storage (more details [here][datastore]). + +A `DatastoreCluster` (sometimes referred to as StoragePod) is a logical grouping of `Datastores`, it provides some resource management capabilities (more details [here][datastore_cluster]). + +VSphere provider configuration in a `MachineDeployment` should specify either a `Datastore` or a `DatastoreCluster`. If both are specified or if one of the two is missing the `MachineDeployment` validation will fail. + +*Note that* +the `datastore` or `datastoreCluster` specified in the `MachineDeployment` will be only used for the placement of VM and disk files related to the VMs provisioned by the `machine controller`. They do not influence the placement of persistent volumes used by PODs, that only depends on the cloud configuration given to the k8s cloud provider running in control plane. -1. Download the CentOS cloud image to your local workstation from here: `https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2` -1. Convert it to vmdk: `qemu-img convert -f qcow2 -O vmdk CentOS-7-x86_64-GenericCloud.qcow2 CentOS-7-x86_64-GenericCloud.vmdk` -1. Upload it to a Datastore of your Vsphere installation -1. Create a new virtual machine that uses the uploaded vmdk as rootdisk +[vm_templates]: https://docs.vmware.com/en/VMware-vSphere/6.7/com.vmware.vsphere.vm_admin.doc/GUID-F7BF0E6B-7C4F-4E46-8BBF-76229AEA7220.html?hWord=N4IghgNiBcIG4FsAEAXApggDhM6DOIAvkA +[datastore]: https://docs.vmware.com/en/VMware-vSphere/6.7/com.vmware.vsphere.storage.doc/GUID-3CC7078E-9C30-402C-B2E1-2542BEE67E8F.html +[datastore_cluster]: https://docs.vmware.com/en/VMware-vSphere/6.7/com.vmware.vsphere.resmgmt.doc/GUID-598DF695-107E-406B-9C95-0AF961FC227A.html +[inflate_thin_virtual_disks]: https://docs.vmware.com/en/VMware-vSphere/6.7/com.vmware.vsphere.storage.doc/GUID-C371B88F-C407-4A69-8F3B-FA877D6955F8.html +[rh_portal_rhel8]: https://access.redhat.com/downloads/content/479/ver=/rhel---8/8.1/x86_64/product-software