From fe134931c63cce5d16aeade6847c529eeddb0289 Mon Sep 17 00:00:00 2001 From: Audrey Spaulding Date: Tue, 16 Sep 2025 13:45:00 -0400 Subject: [PATCH 1/3] CNV-67095 --- ...reating-and-exposing-mediated-devices.adoc | 126 ++++++++++-------- 1 file changed, 68 insertions(+), 58 deletions(-) diff --git a/modules/virt-creating-and-exposing-mediated-devices.adoc b/modules/virt-creating-and-exposing-mediated-devices.adoc index 64308d2addc1..151165d2d7b7 100644 --- a/modules/virt-creating-and-exposing-mediated-devices.adoc +++ b/modules/virt-creating-and-exposing-mediated-devices.adoc @@ -1,12 +1,12 @@ // Module included in the following assemblies: // -// * virt/virtual_machines/advanced_vm_management/virt-configuring-virtual-gpus.adoc +// * virt/managing_vms/advanced_vm_management/virt-configuring-virtual-gpus.adoc :_mod-docs-content-type: PROCEDURE [id="virt-creating-exposing-mediated-devices_{context}"] = Creating and exposing mediated devices -As an administrator, you can create mediated devices and expose them to the cluster by editing the `HyperConverged` custom resource (CR). +As an administrator, you can create mediated devices and expose them to the cluster by editing the `HyperConverged` custom resource (CR). The mediated device values that you supply can vary depending on the particular Graphics Processing Units (GPUs) you are using. .Prerequisites @@ -15,6 +15,11 @@ As an administrator, you can create mediated devices and expose them to the clus * If your hardware vendor provides drivers, you installed them on the nodes where you want to create mediated devices. ** If you use NVIDIA cards, you link:https://docs.nvidia.com/datacenter/cloud-native/openshift/latest/openshift-virtualization.html[installed the NVIDIA GRID driver]. +[IMPORTANT] +==== +Before {VirtProductName} 4.14, the `mediatedDeviceTypes` field was named `mediatedDevicesTypes`. Ensure that you use the correct field name when configuring mediated devices. +==== + .Procedure . Open the `HyperConverged` CR in your default editor by running the following command: @@ -24,7 +29,7 @@ As an administrator, you can create mediated devices and expose them to the clus $ oc edit hyperconverged kubevirt-hyperconverged -n {CNVNamespace} ---- + -.Example configuration file with mediated devices configured +.Example configuration [%collapsible] ==== [source,yaml,subs="attributes+"] @@ -43,81 +48,86 @@ spec: - nvidia-233 nodeSelector: kubernetes.io/hostname: node-11.redhat.com - permittedHostDevices: - mediatedDevices: - - mdevNameSelector: GRID T4-2Q - resourceName: nvidia.com/GRID_T4-2Q - - mdevNameSelector: GRID T4-8Q - resourceName: nvidia.com/GRID_T4-8Q -# ... + # ... ---- ==== -. Create mediated devices by adding them to the `spec.mediatedDevicesConfiguration` stanza: -+ -.Example YAML snippet -[source,yaml] ----- -# ... -spec: - mediatedDevicesConfiguration: - mediatedDeviceTypes: <1> - - - nodeMediatedDeviceTypes: <2> - - mediatedDeviceTypes: <3> - - - nodeSelector: <4> - : -# ... ----- -<1> Required: Configures global settings for the cluster. -<2> Optional: Overrides the global configuration for a specific node or group of nodes. Must be used with the global `mediatedDeviceTypes` configuration. -<3> Required if you use `nodeMediatedDeviceTypes`. Overrides the global `mediatedDeviceTypes` configuration for the specified nodes. -<4> Required if you use `nodeMediatedDeviceTypes`. Must include a `key:value` pair. -+ -[IMPORTANT] -==== -Before {VirtProductName} 4.14, the `mediatedDeviceTypes` field was named `mediatedDevicesTypes`. Ensure that you use the correct field name when configuring mediated devices. -==== - -. Identify the name selector and resource name values for the devices that you want to expose to the cluster. You will add these values to the `HyperConverged` CR in the next step. -.. Find the `resourceName` value by running the following command: +. Identify the name selector and resource name values for the devices that you want to expose to the cluster, as shown in the following example. You can use the same value for both, replacing any spaces in the name with an underscore: + [source,terminal] ---- -$ oc get $NODE -o json \ - | jq '.status.allocatable \ - | with_entries(select(.key | startswith("nvidia.com/"))) \ - | with_entries(select(.value != "0"))' +$ oc debug node/node-11.redhat.com +sh-5.1# chroot /host +sh-5.1# cd sys/class/mdev_bus +sh-5.1# ls +0000:4b:00.4 +sh-5.1# cd 0000:4b:00.4/mdev_supported_types +sh-5.1# ls +nvidia-742 nvidia-744 nvidia-746 nvidia-748 nvidia-750 nvidia-752 +nvidia-743 nvidia-745 nvidia-747 nvidia-749 nvidia-751 nvidia-753 +sh-5.1# cd nvidia-745 +sh-5.1# ls +available_instances create description device_api devices name +sh-5.1# cat name +NVIDIA A2-2Q ---- -.. Find the `mdevNameSelector` value by viewing the contents of `/sys/bus/pci/devices/::./mdev_supported_types//name`, substituting the correct values for your system. -+ -For example, the name file for the `nvidia-231` type contains the selector string `GRID T4-2Q`. Using `GRID T4-2Q` as the `mdevNameSelector` value allows nodes to use the `nvidia-231` type. +. As shown in the following example: +.. Create mediated devices by adding them to the `spec.mediatedDevicesConfiguration` stanza. -. Expose the mediated devices to the cluster by adding the `mdevNameSelector` and `resourceName` values to the -`spec.permittedHostDevices.mediatedDevices` stanza of the `HyperConverged` CR: +.. Expose the mediated devices to the cluster by adding the `mdevNameSelector` and `resourceName` values to the `spec.permittedHostDevices.mediatedDevices` stanza of the `HyperConverged` CR: + -.Example YAML snippet +.Example snippet [source,yaml] ---- -# ... - permittedHostDevices: - mediatedDevices: - - mdevNameSelector: GRID T4-2Q <1> - resourceName: nvidia.com/GRID_T4-2Q <2> -# ... +spec: + mediatedDevicesConfiguration: + mediatedDeviceTypes: + - nvidia-745 + nodeMediatedDeviceTypes: + - mediatedDeviceTypes: + - nvidia-746 + nodeSelector: + kubernetes.io/hostname: cnv-qe-infra-03.cnvqe3.lab.eng.rdu2.redhat.com + permittedHostDevices: + mediatedDevices: + - mdevNameSelector: GRID A2-2Q + resourceName: nvidia.com/GRID_A2-2Q + - mdevNameSelector: GRID A2-4Q + resourceName: nvidia.com/GRID_A2-4Q ---- -<1> Exposes the mediated devices that map to this value on the host. -<2> Matches the resource name that is allocated on the node. ++ +where: + +:: Specifies global settings for the cluster and is required. + +:: Specifies global configuration overrides for a specific node or group of nodes and is optional. Must be used with the global `mediatedDeviceTypes` configuration. + +:: Specifies an override to the global `mediatedDeviceTypes` configuration for the specified nodes. Required if you use `nodeMediatedDeviceTypes`. + +:: Specifies the node selector and must include a `key:value` pair. Required if you use `nodeMediatedDeviceTypes`. + +:: Specifies the mediated devices that map to this value on the host. + +:: Specifies the matching resource name that is allocated on the node. . Save your changes and exit the editor. .Verification -* Optional: Confirm that a device was added to a specific node by running the following command: +* Optional: Verify that a device was added to a specific node by running the following command: + [source,terminal] ---- $ oc describe node ---- + +* Confirm that the virtual GPU is attached to the node by running the following command: ++ +[source,terminal] +---- +$ oc get node -o json \ + | jq '.status.allocatable + | with_entries(select(.key | startswith("nvidia.com/"))) + | with_entries(select(.value != "0"))' +---- From c1774a1691b4f2fa60993e6afbbd9c302c16d82f Mon Sep 17 00:00:00 2001 From: Audrey Spaulding Date: Fri, 3 Oct 2025 10:23:07 -0400 Subject: [PATCH 2/3] additional input from QE --- modules/virt-creating-and-exposing-mediated-devices.adoc | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/modules/virt-creating-and-exposing-mediated-devices.adoc b/modules/virt-creating-and-exposing-mediated-devices.adoc index 151165d2d7b7..a26c593151b0 100644 --- a/modules/virt-creating-and-exposing-mediated-devices.adoc +++ b/modules/virt-creating-and-exposing-mediated-devices.adoc @@ -88,7 +88,7 @@ spec: - mediatedDeviceTypes: - nvidia-746 nodeSelector: - kubernetes.io/hostname: cnv-qe-infra-03.cnvqe3.lab.eng.rdu2.redhat.com + kubernetes.io/hostname: node-11.redhat.com permittedHostDevices: mediatedDevices: - mdevNameSelector: GRID A2-2Q @@ -115,13 +115,6 @@ where: .Verification -* Optional: Verify that a device was added to a specific node by running the following command: -+ -[source,terminal] ----- -$ oc describe node ----- - * Confirm that the virtual GPU is attached to the node by running the following command: + [source,terminal] From 60702dde5ba668a58c713b2a5a91f172f25e6bbb Mon Sep 17 00:00:00 2001 From: vathakur Date: Mon, 13 Oct 2025 12:52:07 +0530 Subject: [PATCH 3/3] added changes --- ...reating-and-exposing-mediated-devices.adoc | 42 +++++++++++-------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/modules/virt-creating-and-exposing-mediated-devices.adoc b/modules/virt-creating-and-exposing-mediated-devices.adoc index a26c593151b0..112a298023f8 100644 --- a/modules/virt-creating-and-exposing-mediated-devices.adoc +++ b/modules/virt-creating-and-exposing-mediated-devices.adoc @@ -11,9 +11,9 @@ As an administrator, you can create mediated devices and expose them to the clus .Prerequisites * You have installed the {oc-first}. -* You enabled the Input-Output Memory Management Unit (IOMMU) driver. -* If your hardware vendor provides drivers, you installed them on the nodes where you want to create mediated devices. -** If you use NVIDIA cards, you link:https://docs.nvidia.com/datacenter/cloud-native/openshift/latest/openshift-virtualization.html[installed the NVIDIA GRID driver]. +* You have enabled the Input-Output Memory Management Unit (IOMMU) driver. +* If your hardware vendor provides drivers, you have installed them on the nodes where you want to create mediated devices. +** If you use NVIDIA cards, you have link:https://docs.nvidia.com/datacenter/cloud-native/openshift/latest/openshift-virtualization.html[installed the NVIDIA GRID driver]. [IMPORTANT] ==== @@ -29,10 +29,10 @@ Before {VirtProductName} 4.14, the `mediatedDeviceTypes` field was named `mediat $ oc edit hyperconverged kubevirt-hyperconverged -n {CNVNamespace} ---- + -.Example configuration +*Example configuration* [%collapsible] -==== -[source,yaml,subs="attributes+"] + +[source,yaml] ---- apiVersion: hco.kubevirt.io/v1 kind: HyperConverged @@ -50,9 +50,8 @@ spec: kubernetes.io/hostname: node-11.redhat.com # ... ---- -==== -. Identify the name selector and resource name values for the devices that you want to expose to the cluster, as shown in the following example. You can use the same value for both, replacing any spaces in the name with an underscore: + Identify the name selector and resource name values for the devices that you want to expose to the cluster, as shown in the following example. You can use the same value for both, replacing any spaces in the name with an underscore. + [source,terminal] ---- @@ -60,22 +59,29 @@ $ oc debug node/node-11.redhat.com sh-5.1# chroot /host sh-5.1# cd sys/class/mdev_bus sh-5.1# ls -0000:4b:00.4 sh-5.1# cd 0000:4b:00.4/mdev_supported_types sh-5.1# ls -nvidia-742 nvidia-744 nvidia-746 nvidia-748 nvidia-750 nvidia-752 -nvidia-743 nvidia-745 nvidia-747 nvidia-749 nvidia-751 nvidia-753 sh-5.1# cd nvidia-745 sh-5.1# ls -available_instances create description device_api devices name sh-5.1# cat name +---- + +.Example output +[source,terminal] +---- +0000:4b:00.4 +nvidia-742 nvidia-744 nvidia-746 nvidia-748 nvidia-750 nvidia-752 +nvidia-743 nvidia-745 nvidia-747 nvidia-749 nvidia-751 nvidia-753 +available_instances create description device_api devices name NVIDIA A2-2Q ---- -. As shown in the following example: -.. Create mediated devices by adding them to the `spec.mediatedDevicesConfiguration` stanza. +. Create and expose mediated devices by: +.. Adding them to the `spec.mediatedDevicesConfiguration` stanza. +.. Adding the `mdevNameSelector` and `resourceName` values to the `spec.permittedHostDevices.mediatedDevices` stanza of the `HyperConverged` CR. -.. Expose the mediated devices to the cluster by adding the `mdevNameSelector` and `resourceName` values to the `spec.permittedHostDevices.mediatedDevices` stanza of the `HyperConverged` CR: +. Identify the `mdevNameSelector` value by viewing the contents of: +`/sys/bus/pci/devices/::./mdev_supported_types//name`. + .Example snippet [source,yaml] @@ -120,7 +126,7 @@ where: [source,terminal] ---- $ oc get node -o json \ - | jq '.status.allocatable - | with_entries(select(.key | startswith("nvidia.com/"))) - | with_entries(select(.value != "0"))' + | jq '.status.allocatable' \ + | with_entries(select(.key | startswith("nvidia.com/"))) \ + | with_entries(select(.value != "0")) ----