Merge pull request #1719 from tkatila/gnrd-qat-support

Add QAT support for 420xx driver and the 4946 device (GNR-D)
intel · May 2, 2024 · d6eb243 · d6eb243
2 parents 6bdd3ac + 78d416f
commit d6eb243
Show file tree

Hide file tree

Showing 11 changed files with 25 additions and 11 deletions.
diff --git a/.github/workflows/lib-e2e.yaml b/.github/workflows/lib-e2e.yaml
@@ -27,6 +27,13 @@ jobs:
               - intel-idxd-config-initcontainer
               - accel-config-demo
               - intel-deviceplugin-operator
+          - name: e2e-qat-gnrd
+            targetjob: e2e-qat FOCUS="Mode:dpdk.*Resource:(cy|dc)" SKIP=App:crypto-perf
+            runner: simics-gnrd
+            images:
+              - intel-qat-plugin
+              - intel-qat-initcontainer
+              - openssl-qat-engine
           - name: e2e-qat
             targetjob: e2e-qat FOCUS=Resource:generic
             runner: qat

diff --git a/cmd/qat_plugin/README.md b/cmd/qat_plugin/README.md
@@ -43,7 +43,7 @@ The QAT plugin can take a number of command line arguments, summarised in the fo
 | Flag | Argument | Meaning |
 |:---- |:-------- |:------- |
 | -dpdk-driver | string | DPDK Device driver for configuring the QAT device (default: `vfio-pci`) |
-| -kernel-vf-drivers | string | Comma separated list of the QuickAssist VFs to search and use in the system. Devices supported: DH895xCC, C62x, C3xxx, 4xxx/401xx/402xx, C4xxx and D15xx (default: `c6xxvf,4xxxvf`) |
+| -kernel-vf-drivers | string | Comma separated list of the QuickAssist VFs to search and use in the system. Devices supported: DH895xCC, C62x, C3xxx, 4xxx/401xx/402xx, 420xx, C4xxx and D15xx (default: `c6xxvf,4xxxvf,420xxvf`) |
 | -max-num-devices | int | maximum number of QAT devices to be provided to the QuickAssist device plugin (default: `64`) |
 | -mode | string | Deprecated: plugin mode which can be either `dpdk` or `kernel` (default: `dpdk`).|
 | -allocation-policy | string | 2 possible values: balanced and packed. Balanced mode spreads allocated QAT VF resources balanced among QAT PF devices, and packed mode packs one QAT PF device full of QAT VF resources before allocating resources from the next QAT PF. (There is no default.) |
@@ -132,7 +132,7 @@ In addition to the default configuration, you can add device-specific configurat
 
 | Device | Possible Configuration | How To Customize | Options | Notes |
 |:-------|:-----------------------|:-----------------|:--------|:------|
-| 4xxx, 401xx,402xx | [cfg_services](https://github.com/torvalds/linux/blob/v6.6-rc5/Documentation/ABI/testing/sysfs-driver-qat) reports the configured services (crypto services or compression services) of the QAT device. | `ServicesEnabled=<value>` | compress:`dc`, crypto:`sym;asym`, <br>crypto+compress:`asym;dc`,<br>crypto+compress:`sym;dc` | Linux 6.0+ kernel is required. |
+| 4xxx, 401xx, 402xx, 420xx | [cfg_services](https://github.com/torvalds/linux/blob/v6.6-rc5/Documentation/ABI/testing/sysfs-driver-qat) reports the configured services (crypto services or compression services) of the QAT device. | `ServicesEnabled=<value>` | compress:`dc`, crypto:`sym;asym`, <br>crypto+compress:`asym;dc`,<br>crypto+compress:`sym;dc` | 4xxx/401xx/402xx: Linux 6.0+ kernel. 420xx: Linux 6.8+ kernel. |
 
 To create a provisioning `configMap`, run the following command before deploying initcontainer:
 
@@ -247,7 +247,7 @@ In order to utilise the QAT device plugin, QuickAssist SR-IOV virtual functions
 You can verify this on your nodes by checking for the relevant PCI identifiers:
 
 ```bash
-for i in 0442 0443 18a1 37c9 6f55 19e3 4941 4943; do lspci -d 8086:$i; done
+for i in 0442 0443 18a1 37c9 6f55 19e3 4941 4943 4945 4947; do lspci -d 8086:$i; done
 ```
 
 [1]:https://www-ssl.intel.com/content/www/us/en/design/products-and-solutions/processors-and-chipsets/purley/intel-xeon-scalable-processors.html

diff --git a/cmd/qat_plugin/dpdkdrv/dpdkdrv.go b/cmd/qat_plugin/dpdkdrv/dpdkdrv.go
@@ -66,6 +66,7 @@ var qatDeviceDriver = map[string]string{
 	"4941": "4xxxvf",
 	"4943": "4xxxvf",
 	"4945": "4xxxvf",
+	"4947": "420xxvf",
 	"37c9": "c6xxvf",
 	"6f55": "d15xxvf",
 }
@@ -382,7 +383,7 @@ func readDeviceConfiguration(pfDev string) string {
 	}
 
 	devCfgPath := filepath.Join(filepath.Dir(filepath.Join(pfDev, "../../")), "kernel/debug",
-		fmt.Sprintf("qat_4xxx_%s/dev_cfg", filepath.Base(pfDev)))
+		fmt.Sprintf("qat_%s_%s/dev_cfg", getCurrentDriver(pfDev), filepath.Base(pfDev)))
 
 	devCfg, err := ini.LoadSources(lOpts, devCfgPath)
 	if err != nil {
@@ -434,6 +435,7 @@ func getDeviceCapabilities(device string) (string, error) {
 		"4941": {}, // QAT Gen4 (4xxx) VF PCI ID
 		"4943": {}, // QAT Gen4 (401xx) VF PCI ID
 		"4945": {}, // QAT Gen4 (402xx) VF PCI ID
+		"4947": {}, // QAT Gen4 (420xx) VF PCI ID
 	}
 
 	if _, ok := devicesWithCapabilities[devID]; !ok {

diff --git a/cmd/qat_plugin/qat_plugin.go b/cmd/qat_plugin/qat_plugin.go
@@ -40,7 +40,7 @@ func main() {
 	mode := flag.String("mode", "dpdk", "plugin mode which can be either dpdk (default) or kernel")
 
 	dpdkDriver := flag.String("dpdk-driver", "vfio-pci", "DPDK Device driver for configuring the QAT device")
-	kernelVfDrivers := flag.String("kernel-vf-drivers", "c6xxvf,4xxxvf", "Comma separated VF Device Driver of the QuickAssist Devices in the system. Devices supported: DH895xCC, C62x, C3xxx, C4xxx, 4xxx, and D15xx")
+	kernelVfDrivers := flag.String("kernel-vf-drivers", "c6xxvf,4xxxvf,420xxvf", "Comma separated VF Device Driver of the QuickAssist Devices in the system. Devices supported: DH895xCC, C62x, C3xxx, C4xxx, 4xxx, 420xxx, and D15xx")
 	preferredAllocationPolicy := flag.String("allocation-policy", "", "Modes of allocating QAT devices: balanced and packed")
 	maxNumDevices := flag.Int("max-num-devices", 64, "maximum number of QAT devices to be provided to the QuickAssist device plugin")
 	flag.Parse()

diff --git a/demo/qat-init.sh b/demo/qat-init.sh
@@ -1,12 +1,13 @@
 #!/usr/bin/env bash
 # This script is based on qatlib's qat_init.sh
 NODE_NAME="${NODE_NAME:-}"
-ENABLED_QAT_PF_PCIIDS=${ENABLED_QAT_PF_PCIIDS:-37c8 4940 4942 4944}
+ENABLED_QAT_PF_PCIIDS=${ENABLED_QAT_PF_PCIIDS:-37c8 4940 4942 4944 4946}
 DEVS=$(for pf in $ENABLED_QAT_PF_PCIIDS; do lspci -n | grep -e "$pf" | grep -o -e "^\\S*"; done)
 SERVICES_LIST="sym asym sym;asym dc sym;dc asym;dc"
 QAT_4XXX_DEVICE_PCI_ID="0x4940"
 QAT_401XX_DEVICE_PCI_ID="0x4942"
 QAT_402XX_DEVICE_PCI_ID="0x4944"
+QAT_420XX_DEVICE_PCI_ID="0x4946"
 SERVICES_ENABLED="NONE"
 SERVICES_ENABLED_FOUND="FALSE"
 
@@ -31,7 +32,7 @@ sysfs_config() {
     for dev in $DEVS; do
       DEVPATH="/sys/bus/pci/devices/0000:$dev"
       PCI_DEV=$(cat "$DEVPATH"/device 2> /dev/null)
-      if [ "$PCI_DEV" != "$QAT_4XXX_DEVICE_PCI_ID" ] && [ "$PCI_DEV" != "$QAT_401XX_DEVICE_PCI_ID" ] && [ "$PCI_DEV" != "$QAT_402XX_DEVICE_PCI_ID" ]; then
+      if [ "$PCI_DEV" != "$QAT_4XXX_DEVICE_PCI_ID" ] && [ "$PCI_DEV" != "$QAT_401XX_DEVICE_PCI_ID" ] && [ "$PCI_DEV" != "$QAT_402XX_DEVICE_PCI_ID" ] && [ "$PCI_DEV" != "$QAT_420XX_DEVICE_PCI_ID" ]; then
         continue
       fi
 

diff --git a/deployments/nfd/overlays/node-feature-rules/node-feature-rules.yaml b/deployments/nfd/overlays/node-feature-rules/node-feature-rules.yaml
@@ -95,7 +95,7 @@ spec:
         - feature: pci.device
           matchExpressions:
             vendor: {op: In, value: ["8086"]}
-            device: {op: In, value: ["37c8", "4940", "4942", "4944"]}
+            device: {op: In, value: ["37c8", "4940", "4942", "4944", "4946"]}
             class: {op: In, value: ["0b40"]}
         - feature: kernel.loadedmodule
           matchExpressions:

diff --git a/deployments/operator/crd/bases/deviceplugin.intel.com_qatdeviceplugins.yaml b/deployments/operator/crd/bases/deviceplugin.intel.com_qatdeviceplugins.yaml
@@ -81,6 +81,7 @@ spec:
                   - c3xxxvf
                   - d15xxvf
                   - 4xxxvf
+                  - 420xxvf
                   - c4xxxvf
                   type: string
                 type: array

diff --git a/deployments/operator/samples/deviceplugin_v1_qatdeviceplugin.yaml b/deployments/operator/samples/deviceplugin_v1_qatdeviceplugin.yaml
@@ -15,6 +15,7 @@ spec:
   kernelVfDrivers:
     - c6xxvf
     - 4xxxvf
+    - 420xxvf
   maxNumDevices: 1
   logLevel: 4
   nodeSelector:

diff --git a/pkg/apis/deviceplugin/v1/qatdeviceplugin_types.go b/pkg/apis/deviceplugin/v1/qatdeviceplugin_types.go
@@ -21,7 +21,7 @@ import (
 
 // NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
 
-// +kubebuilder:validation:Enum={"dh895xccvf","c6xxvf","c3xxxvf","d15xxvf","4xxxvf","c4xxxvf"}
+// +kubebuilder:validation:Enum={"dh895xccvf","c6xxvf","c3xxxvf","d15xxvf","4xxxvf", "420xxvf", "c4xxxvf"}
 
 // KernelVfDriver is a VF device driver for QuickAssist devices.
 type KernelVfDriver string

diff --git a/pkg/apis/deviceplugin/v1/qatdeviceplugin_webhook.go b/pkg/apis/deviceplugin/v1/qatdeviceplugin_webhook.go
@@ -92,7 +92,8 @@ func (r *QatDevicePlugin) validatePlugin() error {
 		// check if 4xxxvf is enabled
 		contains := false
 		devicesWithCapabilities := map[KernelVfDriver]struct{}{
-			"4xxxvf": {},
+			"4xxxvf":  {},
+			"420xxvf": {},
 		}
 
 		for _, kernelVfDriver := range r.Spec.KernelVfDrivers {
@@ -103,7 +104,7 @@ func (r *QatDevicePlugin) validatePlugin() error {
 		}
 
 		if !contains {
-			return errors.Errorf("ProvisioningConfig is available only for 4xxx devices")
+			return errors.Errorf("ProvisioningConfig is available only for 4xxx and 420xx devices")
 		}
 	}
 

diff --git a/pkg/controllers/qat/controller.go b/pkg/controllers/qat/controller.go
@@ -213,6 +213,7 @@ func setInitContainer(dsSpec *v1.PodSpec, dpSpec devicepluginv1.QatDevicePluginS
 		"c6xxvf":     "37c8",
 		"d15xxvf":    "6f54",
 		"4xxxvf":     "4940 4942 4944",
+		"420xxvf":    "4946",
 		"c4xxxvf":    "18a0",
 	}