From 87e59da523850228bb616f97f9f03fd0b668a25d Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Thu, 2 Apr 2020 10:56:48 +0200 Subject: [PATCH 1/3] add documentation for rhel os and kubevirt cloud provider Signed-off-by: Moath Qasim Signed-off-by: Moath Qasim --- docs/kubevirt.md | 30 ++++++++++++++++++--- docs/rhel-custom-image.md | 57 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 docs/rhel-custom-image.md diff --git a/docs/kubevirt.md b/docs/kubevirt.md index 52bd0bda5..961486462 100644 --- a/docs/kubevirt.md +++ b/docs/kubevirt.md @@ -8,7 +8,29 @@ are some things you need to keep in mind: * The machine-controller will create `VMIs` that have the same name as the underlying `machine`. To avoid collisions, use one namespace per cluster that runs the `machine-controller` -* Service CIDR range: The CIDR ranges of the cluster that runs Kubevirt and the cluster that hosts the machine-controller must not overlap, otherwise routing of services that run in the kubevirt cluster - wont work anymore. THis is especially important for the DNS ClusterIP. -* In order to create VirtualMachineInstances deployed with CoreOS, you should use `coreos_production_qemu_image.img.bz2`, which can be found in the stable releases -[here](https://stable.release.core-os.net/amd64-usr/). \ No newline at end of file +* Service CIDR range: The CIDR ranges of the cluster that runs Kubevirt and the cluster that hosts the machine-controller must not overlap, +otherwise routing of services that run in the kubevirt cluster wont work anymore. THis is especially important for the DNS ClusterIP. +* In order to create VirtualMachineInstances deployed with CoreOS, you should use `coreos_production_qemu_image.img.bz2`, which can be found in +the stable releases [here](https://stable.release.core-os.net/amd64-usr/). + +## Serving Supported Images + +For KubeVirt clusters, we use Containerized Data Importer (CDI), which is is a utility to import, upload and clone +Virtual Machine images for use with KubeVirt. At a high level, a persistent volume claim (PVC), which defines VM-suitable +storage via a storage class, is created. + +The Containerized Data Importer is capable of performing certain functions that streamline its use with KubeVirt. It automatically +decompresses gzip and xz files, and un-tar’s tar archives. Also, qcow2 images are converted into the raw format which is required by KubeVirt, +resulting in the final file being a simple .img file. + +Supported file formats are: + +- Tar archive +- Gzip compressed file +- XZ compressed file +- Raw image data +- ISO image data +- Qemu qcow2 image data + +KubeVirt reads those images from an http endpoint which is passed to the `MachineDeployment` spec. The field that should be used +for to import those images is `sourceURL`. \ No newline at end of file diff --git a/docs/rhel-custom-image.md b/docs/rhel-custom-image.md new file mode 100644 index 000000000..c25036604 --- /dev/null +++ b/docs/rhel-custom-image.md @@ -0,0 +1,57 @@ +# RedHat Enterprise Linux + +Cloud providers which are listed below, support using rhel as an operating system option: + +- AWS +- Azure +- GCE +- KubeVirt +- Openstack +- vSphere + +#### AWS: +For amazon web service cloud provider, one the user applies for Gold Accesses Images using their RedHat Customer Portal, their request +will be reviewed and validated. Afterwards, new images will be added to the aws account under EC2-> Images-> AMIs-> Private Images. +Once the images are available in the aws account, the image id for rhel(at the moment we only support 8.x.x versions) should be then +added to the `MachineDeployment` spec to the field `ami`. + +#### Azure +Same for aws, RedHat supports Azure and provides images for it. Once those images are available, the field `imageID` in the +`MachineDeployment` spec should be updated with this image. + +**Note:** +Azure rhel images starting from 7.6.x don't support cloud-init as their documentation states [here](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/using-cloud-init#rhel). +Thus, custom images can be used with a cloud-init pre-installed to solve this issue. Follow this [documentation](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/cloudinit-prepare-custom-image) +to prepare an image with cloud-init support. + +#### GCE +RedHat also provides Gold Access Image for GCE and those can be fetched just like aws and azure. The image id should be added to +the `MachineDeployment` spec in the field `customImage`. + +**Note:** +Same as for Azure, rhel images in GCE don't support cloud-init. Thus, custom images can be used with a cloud-init pre-installed +to solve this issue. Follow this [documentation](https://cloud.google.com/compute/docs/images/export-image) to upload custom rhel +images in order to use it for running rhel instances. + +#### KubeVirt +In order to create machines which run rhel as an operating system in KubeVirt cloud provider, the image should be available and fetched +via an endpoint. This endpoint should be then added to the `Machinedeployment` spec in the field `sourceURL`. For more information about +the supported images please refer to this documentation from KubeVirt CDI [here](https://kubevirt.io/2018/containerized-data-importer.html) + +#### Openstack +Once RHEL images(e.g: Red Hat Enterprise Linux 8.x KVM Guest Image) is uploaded to openstack, the image name should be used in +the `MachineDeployment` spec in the filed `image`. + +#### vSphere +To rhel os for vSphere instance, a template for the rhel machine should be created or a clone from a rhel machine. To upload rhel +image to vSphere, follow these steps to create instances from a cloned machine: + +- Download Red Hat Enterprise Linux 8.x KVM Guest Image from Red Hat Customer Portal. +- The image has the format `qcow2` thus should be converted to `vmdk` by running the command: `qemu-img convert -f qcow2 rhel.qcow2 -O vmdk newRHEL.vmdk` +- Run `vmkfstools -i myImage.vmdk outputName.vmdk -d thin To ensure that the` to ensure that, the `vmdk` is ESXi compatible. +- Upload the image to vSphere Datastore. Preferably use [`govc`](https://github.com/vmware/govmomi/blob/master/govc/USAGE.md#datastoreupload) +- Once the image is uploaded create a new instance using that image. During the machine creation process, at the `Customize Hardware` +step, press on ADD NEW DEVICE and select Existing Hard Disk. +- In the Existing Hard Disk wizard select the rhel image file and then create the instance. +- Once the instance and everything is working as expected, power it off. +- Use the instance name to clone rhel machine by updating the `MachineDeployment` spec and the name in the `templateVMName` field. From 20e1bd6ef9453720b5c139558467df3a6598c807 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Thu, 2 Apr 2020 13:31:50 +0200 Subject: [PATCH 2/3] addressing PR review Signed-off-by: Moath Qasim Signed-off-by: Moath Qasim --- docs/kubevirt.md | 2 +- docs/rhel-custom-image.md | 24 +++++++++++------------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/docs/kubevirt.md b/docs/kubevirt.md index 961486462..0c6b13342 100644 --- a/docs/kubevirt.md +++ b/docs/kubevirt.md @@ -9,7 +9,7 @@ are some things you need to keep in mind: * The machine-controller will create `VMIs` that have the same name as the underlying `machine`. To avoid collisions, use one namespace per cluster that runs the `machine-controller` * Service CIDR range: The CIDR ranges of the cluster that runs Kubevirt and the cluster that hosts the machine-controller must not overlap, -otherwise routing of services that run in the kubevirt cluster wont work anymore. THis is especially important for the DNS ClusterIP. +otherwise routing of services that run in the kubevirt cluster wont work anymore. This is especially important for the DNS ClusterIP. * In order to create VirtualMachineInstances deployed with CoreOS, you should use `coreos_production_qemu_image.img.bz2`, which can be found in the stable releases [here](https://stable.release.core-os.net/amd64-usr/). diff --git a/docs/rhel-custom-image.md b/docs/rhel-custom-image.md index c25036604..948460a1c 100644 --- a/docs/rhel-custom-image.md +++ b/docs/rhel-custom-image.md @@ -10,14 +10,14 @@ Cloud providers which are listed below, support using rhel as an operating syste - vSphere #### AWS: -For amazon web service cloud provider, one the user applies for Gold Accesses Images using their RedHat Customer Portal, their request -will be reviewed and validated. Afterwards, new images will be added to the aws account under EC2-> Images-> AMIs-> Private Images. -Once the images are available in the aws account, the image id for rhel(at the moment we only support 8.x.x versions) should be then -added to the `MachineDeployment` spec to the field `ami`. +For amazon web service cloud provider, First of all the RHEL gold image AMIs have to be enabled from the +[RedHat Customer Portal](https://access.redhat.com/public-cloud/aws) (this requires a [cloud-provider subscription](https://access.redhat.com/public-cloud)). +.Afterwards, new images will be added to the aws account under EC2-> Images-> AMIs-> Private Images. Once the images are available in the aws account, +the image id for rhel(at the moment we only support 8.x.x versions) should be then added to the `MachineDeployment` spec to the field `ami`. #### Azure -Same for aws, RedHat supports Azure and provides images for it. Once those images are available, the field `imageID` in the -`MachineDeployment` spec should be updated with this image. +RedHat provides images for Azure, [documentation](https://access.redhat.com/articles/uploading-rhel-image-to-azure) is available on RH customer portal. +The `MachineDeployment` field `.spec.template.spec.providerSpec.value.cloudProviderSpec.imageID` should reference the ID of the uploaded VM. **Note:** Azure rhel images starting from 7.6.x don't support cloud-init as their documentation states [here](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/using-cloud-init#rhel). @@ -25,22 +25,21 @@ Thus, custom images can be used with a cloud-init pre-installed to solve this is to prepare an image with cloud-init support. #### GCE -RedHat also provides Gold Access Image for GCE and those can be fetched just like aws and azure. The image id should be added to -the `MachineDeployment` spec in the field `customImage`. +RedHat also provides Gold Access Image for GCE and those can be fetched just like aws and azure. The `MachineDeployment` field `.spec.template.spec.providerSpec.value.cloudProviderSpec.customImage` should reference the ID of the used image. **Note:** Same as for Azure, rhel images in GCE don't support cloud-init. Thus, custom images can be used with a cloud-init pre-installed -to solve this issue. Follow this [documentation](https://cloud.google.com/compute/docs/images/export-image) to upload custom rhel +to solve this issue. Follow this [documentation](https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/8/html/deploying_red_hat_enterprise_linux_8_on_public_cloud_platforms/assembly_deploying-a-rhel-image-as-a-compute-engine-instance-on-google-cloud-platform_deploying-a-virtual-machine-on-aws) to upload custom rhel images in order to use it for running rhel instances. #### KubeVirt In order to create machines which run rhel as an operating system in KubeVirt cloud provider, the image should be available and fetched -via an endpoint. This endpoint should be then added to the `Machinedeployment` spec in the field `sourceURL`. For more information about +via an endpoint. This endpoint should be then added to the `MachineDeployment` field `.spec.template.spec.providerSpec.value.cloudProviderSpec.sourceURL`. For more information about the supported images please refer to this documentation from KubeVirt CDI [here](https://kubevirt.io/2018/containerized-data-importer.html) #### Openstack Once RHEL images(e.g: Red Hat Enterprise Linux 8.x KVM Guest Image) is uploaded to openstack, the image name should be used in -the `MachineDeployment` spec in the filed `image`. +the `MachineDeployment` field `.spec.template.spec.providerSpec.value.cloudProviderSpec.image`. #### vSphere To rhel os for vSphere instance, a template for the rhel machine should be created or a clone from a rhel machine. To upload rhel @@ -53,5 +52,4 @@ image to vSphere, follow these steps to create instances from a cloned machine: - Once the image is uploaded create a new instance using that image. During the machine creation process, at the `Customize Hardware` step, press on ADD NEW DEVICE and select Existing Hard Disk. - In the Existing Hard Disk wizard select the rhel image file and then create the instance. -- Once the instance and everything is working as expected, power it off. -- Use the instance name to clone rhel machine by updating the `MachineDeployment` spec and the name in the `templateVMName` field. +- Use the instance name to clone rhel machine by updating the `MachineDeployment` field `.spec.template.spec.providerSpec.value.cloudProviderSpec.templateVMName`. From 7741577bd8d37eb4df07c5fab87b7f061d671378 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Thu, 2 Apr 2020 14:58:41 +0200 Subject: [PATCH 3/3] updating machine-deployment manifests Signed-off-by: Moath Qasim Signed-off-by: Moath Qasim --- examples/alibaba-machinedeployment.yaml | 5 ++++- examples/aws-machinedeployment.yaml | 4 +++- examples/azure-machinedeployment.yaml | 5 ++++- examples/gce-machinedeployment.yaml | 3 ++- examples/openstack-machinedeployment.yaml | 1 + examples/vsphere-datastore-cluster-machinedeployment.yaml | 1 + examples/vsphere-machinedeployment.yaml | 1 + .../machinedeployment-vsphere-datastore-cluster.yaml | 3 +++ 8 files changed, 19 insertions(+), 4 deletions(-) diff --git a/examples/alibaba-machinedeployment.yaml b/examples/alibaba-machinedeployment.yaml index 5e808efa9..603092b31 100644 --- a/examples/alibaba-machinedeployment.yaml +++ b/examples/alibaba-machinedeployment.yaml @@ -51,12 +51,15 @@ spec: key: accessKeySecret instanceType: "ecs.t1.xsmall" instanceName: "alibaba-instance" + internetMaxBandwidthOut: 10 regionID: eu-central-1 - imageID: "aliyun_2_1903_64_20G_alibase_20190829.vhd" + vSwitchID: "vswitchID" + zoneID: eu-central-1a diskType: "cloud_efficiency" diskSize: "40" operatingSystem: "ubuntu" operatingSystemSpec: distUpgradeOnBoot: false + disableAutoUpdate: true versions: kubelet: 1.13.1 diff --git a/examples/aws-machinedeployment.yaml b/examples/aws-machinedeployment.yaml index 66874265e..04320a097 100644 --- a/examples/aws-machinedeployment.yaml +++ b/examples/aws-machinedeployment.yaml @@ -59,16 +59,18 @@ spec: diskSize: 50 diskType: "gp2" ebsVolumeEncrypted: false + ami: "my-custom-ami" ## Only application if diskType = io1 diskIops: 500 # Assign a public IP to this instance. Default: true assignPublicIP: false tags: "KubernetesCluster": "6qsm86c2d" - # Can be 'ubuntu', 'coreos' ,'centos' or 'sles' + # Can be 'ubuntu', 'coreos' ,'centos', `rhel` or 'sles' operatingSystem: "coreos" operatingSystemSpec: disableAutoUpdate: true + distUpgradeOnBoot: false # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` diff --git a/examples/azure-machinedeployment.yaml b/examples/azure-machinedeployment.yaml index 476c2458f..c0a3b8f16 100644 --- a/examples/azure-machinedeployment.yaml +++ b/examples/azure-machinedeployment.yaml @@ -64,18 +64,21 @@ spec: key: subscriptionID location: "westeurope" resourceGroup: "<< YOUR_RESOURCE_GROUP >>" - vmSize: "Standard_B1ms" + vmSize: "Standard_F2" # optional disk size values in GB. If not set, the defaults for the vmSize will be used. osDiskSize: 30 dataDiskSize: 30 vnetName: "<< VNET_NAME >>" subnetName: "<< SUBNET_NAME >>" routeTableName: "<< ROUTE_TABLE_NAME >>" + imageID: "myImageID" assignPublicIP: false securityGroupName: my-security-group + # Can be 'ubuntu', 'coreos' ,'centos' or 'rhel' operatingSystem: "coreos" operatingSystemSpec: distUpgradeOnBoot: false + disableAutoUpdate: true # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` diff --git a/examples/gce-machinedeployment.yaml b/examples/gce-machinedeployment.yaml index e4ea91416..ae9279bbb 100644 --- a/examples/gce-machinedeployment.yaml +++ b/examples/gce-machinedeployment.yaml @@ -62,7 +62,8 @@ spec: "kubernetes_cluster": "my-cluster" # Whether to assign a public IP Address. Required for Internet access assignPublicIPAddress: true - # Can be 'ubuntu' or 'coreos' + customImage: "myCustomImage" + # Can be 'ubuntu' 'rhel' or 'coreos' operatingSystem: "coreos" operatingSystemSpec: disableAutoUpdate: true diff --git a/examples/openstack-machinedeployment.yaml b/examples/openstack-machinedeployment.yaml index 981f2c8bb..5ae21b5ea 100644 --- a/examples/openstack-machinedeployment.yaml +++ b/examples/openstack-machinedeployment.yaml @@ -112,6 +112,7 @@ spec: operatingSystem: "ubuntu" operatingSystemSpec: distUpgradeOnBoot: true + disableAutoUpdate: true # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` diff --git a/examples/vsphere-datastore-cluster-machinedeployment.yaml b/examples/vsphere-datastore-cluster-machinedeployment.yaml index 2372e7b65..066d7e59e 100644 --- a/examples/vsphere-datastore-cluster-machinedeployment.yaml +++ b/examples/vsphere-datastore-cluster-machinedeployment.yaml @@ -66,6 +66,7 @@ spec: operatingSystem: "ubuntu" operatingSystemSpec: distUpgradeOnBoot: false + disableAutoUpdate: true # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` diff --git a/examples/vsphere-machinedeployment.yaml b/examples/vsphere-machinedeployment.yaml index ed3dbbd37..060f483d7 100644 --- a/examples/vsphere-machinedeployment.yaml +++ b/examples/vsphere-machinedeployment.yaml @@ -66,6 +66,7 @@ spec: operatingSystem: "ubuntu" operatingSystemSpec: distUpgradeOnBoot: false + disableAutoUpdate: true # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml index d3c664c0b..c47004af4 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml @@ -45,5 +45,8 @@ spec: rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>" + # 'rhsmOfflineToken' if it was provided red hat systems subscriptions will be removed upon machines deletions, and if wasn't + # provided the rhsm will be disabled and any created subscription won't be removed automatically + rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: kubelet: "<< KUBERNETES_VERSION >>"