From 401661e5e5921366daadf7137fe6d597d42044d5 Mon Sep 17 00:00:00 2001 From: damiankopyto Date: Thu, 30 Jan 2020 13:42:25 +0000 Subject: [PATCH 1/2] =?UTF-8?q?SRIOV=20feature=20is=20not=20just=20applica?= =?UTF-8?q?ble=20for=20Network=20interfaces=20but=20any=20IO=20device=20th?= =?UTF-8?q?at=20support=20PCIe=20SIG=20SRIOV=20features.=20One=20such=20de?= =?UTF-8?q?vice=20is=20based=20on=20Intel=C2=AE=20PAC=20N3000=20FPGA=20whi?= =?UTF-8?q?ch=20has=20Network=20Functionality=20of=202x25G=20and=20FPGA=20?= =?UTF-8?q?part=20that=20can=20be=20programmed=20to=20support=20hardware?= =?UTF-8?q?=20offload=20of=20the=20=20Wireless=20(4G/5G)=20Forward=20Error?= =?UTF-8?q?=20Correction=20(FEC).=20This=20Product=20has=20already=20been?= =?UTF-8?q?=20released=20by=20Intel=20and=20is=20being=20used/deployed=20b?= =?UTF-8?q?y=20customers.=20Intel=20FPGA=20Device=20plugin=20(https://gith?= =?UTF-8?q?ub.com/intel/intel-device-plugins-for-kubernetes/blob/master/cm?= =?UTF-8?q?d/fpga=5Fplugin/README.md)=20does=20not=20support=20such=20devi?= =?UTF-8?q?ces=20as=20it=20dose=20not=20cover=20SRIOV=20devices=20using=20?= =?UTF-8?q?DPDK=20and=20also=20differ=20in=20the=20FPGA=20image=20that=20g?= =?UTF-8?q?ets=20programmed.=20As=20part=20of=20this=20PR=20we=20are=20rec?= =?UTF-8?q?ommending=20to=20extend=20the=20SRIOV=20support=20for=20non-net?= =?UTF-8?q?working=20devices=20and=20also=20rename=20the=20device=20plugin?= =?UTF-8?q?=20from=20=E2=80=9Csriov-network-device-plugin=E2=80=9D=20to=20?= =?UTF-8?q?=E2=80=9Csriov-device-plugin=E2=80=9D.=20For=20the=20Intel?= =?UTF-8?q?=C2=AE=20PAC=20N3000=20FPGA=20supporting=20vRAN=20or=20Cloud-na?= =?UTF-8?q?tive=20RAN=20the=20FPGA=20PF=20that=20supports=20FEC=20is=20bou?= =?UTF-8?q?nd=20to=20DPDK=20igb=5Fuio=20or=20vfio=5Fpci.=20The=20Virtual?= =?UTF-8?q?=20Function=20resources=20are=20allocated=20to=20the=20PODs=20r?= =?UTF-8?q?unning=20RAN/FlexRAN=20like=20workload=20and=20uses=20vfio=5Fpc?= =?UTF-8?q?i.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/sriovdp/manager.go | 65 +++++++++++++++++++++++++++++++----------- 1 file changed, 48 insertions(+), 17 deletions(-) diff --git a/cmd/sriovdp/manager.go b/cmd/sriovdp/manager.go index e1f3c2295..5e4dddcf5 100644 --- a/cmd/sriovdp/manager.go +++ b/cmd/sriovdp/manager.go @@ -32,6 +32,7 @@ import ( const ( socketSuffix = "sock" netClass = 0x02 // Device class - Network controller. ref: https://pci-ids.ucw.cz/read/PD/02 (for Sub-Classes) + fecClass = 0x12 // Device class - Processing accelerator ) /* @@ -46,6 +47,7 @@ Network controller subclasses. ref: https://pci-ids.ucw.cz/read/PD/02 07 Infiniband controller 08 Fabric controller 80 Network controller + 12 Processing Accelerator */ type cliParams struct { @@ -194,8 +196,8 @@ func (rm *resourceManager) discoverHostDevices() error { continue } - // only interested in network class - if devClass == netClass { + // only interested in network class + patch for detecting FEC processing accelerator + if devClass == netClass || devClass == fecClass { vendor := device.Vendor vendorName := vendor.Name if len(vendor.Name) > 20 { @@ -208,28 +210,57 @@ func (rm *resourceManager) discoverHostDevices() error { } glog.Infof("discoverDevices(): device found: %-12s\t%-12s\t%-20s\t%-40s", device.Address, device.Class.ID, vendorName, productName) - // exclude device in-use in host - if isDefaultRoute, _ := hasDefaultRoute(device.Address); !isDefaultRoute { + // Network devices + if devClass == netClass { + // exclude device in-use in host + if isDefaultRoute, _ := hasDefaultRoute(device.Address); !isDefaultRoute { - aPF := utils.IsSriovPF(device.Address) - aVF := utils.IsSriovVF(device.Address) + aPF := utils.IsSriovPF(device.Address) + aVF := utils.IsSriovVF(device.Address) - if aPF || !aVF { - // add to linkWatchList - rm.addToLinkWatchList(device.Address) - } + if aPF || !aVF { + // add to linkWatchList + rm.addToLinkWatchList(device.Address) + } + + if aPF && utils.SriovConfigured(device.Address) { + // do not add this device in net device list + continue + } + + if newDevice, err := resources.NewPciNetDevice(device, rm.rFactory); err == nil { + rm.netDeviceList = append(rm.netDeviceList, newDevice) + } else { + glog.Errorf("discoverDevices() error adding new device: %q", err) + } - if aPF && utils.SriovConfigured(device.Address) { - // do not add this device in net device list - continue } + } + + // FEC processing accelerators + if devClass == fecClass { + //Check if device is a physical function (PF) + aPF := utils.IsSriovPF(device.Address) - if newDevice, err := resources.NewPciNetDevice(device, rm.rFactory); err == nil { - rm.netDeviceList = append(rm.netDeviceList, newDevice) + //If dev ID = 0b30 it means it is a RSU device and not FEC + if device.Product.ID != "0b30" { + if aPF && utils.SriovConfigured(device.Address) { + // If it is a PF and SRIOV is configured do not add to device list + continue + } + + if newDevice, err := resources.NewPciNetDevice(device, rm.rFactory); err == nil { + if newDevice.GetDriver() == "igb_uio" || newDevice.GetDriver() == "vfio-pci" { + rm.netDeviceList = append(rm.netDeviceList, newDevice) + } else { + glog.Infof("Excluding FEC device %-12s because it is not bound to userspace driver.", device.Address) + } + } else { + glog.Errorf("discoverDevices() error adding new device: %q", err) + } } else { - glog.Errorf("discoverDevices() error adding new device: %q", err) + glog.Infof("Excluding device %-12s , it is a RSU device.", device.Address) } - } } } From 61c4050ef14af4e234f9ca4f2b64ffc6b098068b Mon Sep 17 00:00:00 2001 From: damiankopyto Date: Fri, 31 Jan 2020 18:10:45 +0000 Subject: [PATCH 2/2] Renaming FEC to ACC (accelerator) to make name more generic. Fixing hardcoded RSU device ID. Placing net dev and acc dev handling into two distinct functions. --- cmd/sriovdp/manager.go | 117 +++++++++++++++++++++++------------------ 1 file changed, 65 insertions(+), 52 deletions(-) diff --git a/cmd/sriovdp/manager.go b/cmd/sriovdp/manager.go index 5e4dddcf5..c6c1cceb6 100644 --- a/cmd/sriovdp/manager.go +++ b/cmd/sriovdp/manager.go @@ -31,8 +31,9 @@ import ( const ( socketSuffix = "sock" - netClass = 0x02 // Device class - Network controller. ref: https://pci-ids.ucw.cz/read/PD/02 (for Sub-Classes) - fecClass = 0x12 // Device class - Processing accelerator + netClass = 0x02 // Device class - Network controller. ref: https://pci-ids.ucw.cz/read/PD/02 (for Sub-Classes) + accClass = 0x12 // Device class - Processing accelerator + rsuDevice = 0x0b30 // RSU device - only used for programming the FPGA ) /* @@ -47,7 +48,7 @@ Network controller subclasses. ref: https://pci-ids.ucw.cz/read/PD/02 07 Infiniband controller 08 Fabric controller 80 Network controller - 12 Processing Accelerator + 12 Processing Accelerator */ type cliParams struct { @@ -196,8 +197,8 @@ func (rm *resourceManager) discoverHostDevices() error { continue } - // only interested in network class + patch for detecting FEC processing accelerator - if devClass == netClass || devClass == fecClass { + // only interested in network class and accelerator class + if devClass == netClass || devClass == accClass { vendor := device.Vendor vendorName := vendor.Name if len(vendor.Name) > 20 { @@ -212,55 +213,12 @@ func (rm *resourceManager) discoverHostDevices() error { // Network devices if devClass == netClass { - // exclude device in-use in host - if isDefaultRoute, _ := hasDefaultRoute(device.Address); !isDefaultRoute { - - aPF := utils.IsSriovPF(device.Address) - aVF := utils.IsSriovVF(device.Address) - - if aPF || !aVF { - // add to linkWatchList - rm.addToLinkWatchList(device.Address) - } - - if aPF && utils.SriovConfigured(device.Address) { - // do not add this device in net device list - continue - } - - if newDevice, err := resources.NewPciNetDevice(device, rm.rFactory); err == nil { - rm.netDeviceList = append(rm.netDeviceList, newDevice) - } else { - glog.Errorf("discoverDevices() error adding new device: %q", err) - } - - } + rm.netDevHandler(device) } - // FEC processing accelerators - if devClass == fecClass { - //Check if device is a physical function (PF) - aPF := utils.IsSriovPF(device.Address) - - //If dev ID = 0b30 it means it is a RSU device and not FEC - if device.Product.ID != "0b30" { - if aPF && utils.SriovConfigured(device.Address) { - // If it is a PF and SRIOV is configured do not add to device list - continue - } - - if newDevice, err := resources.NewPciNetDevice(device, rm.rFactory); err == nil { - if newDevice.GetDriver() == "igb_uio" || newDevice.GetDriver() == "vfio-pci" { - rm.netDeviceList = append(rm.netDeviceList, newDevice) - } else { - glog.Infof("Excluding FEC device %-12s because it is not bound to userspace driver.", device.Address) - } - } else { - glog.Errorf("discoverDevices() error adding new device: %q", err) - } - } else { - glog.Infof("Excluding device %-12s , it is a RSU device.", device.Address) - } + // Processing accelerators + if devClass == accClass { + rm.accDevHandler(device) } } } @@ -321,3 +279,58 @@ type linkWatcher struct { func (lw *linkWatcher) Subscribe() { } + +func (rm *resourceManager) netDevHandler(dev *ghw.PCIDevice) { + // exclude device in-use in host + if isDefaultRoute, _ := hasDefaultRoute(dev.Address); !isDefaultRoute { + + aPF := utils.IsSriovPF(dev.Address) + aVF := utils.IsSriovVF(dev.Address) + + if aPF || !aVF { + // add to linkWatchList + rm.addToLinkWatchList(dev.Address) + } + + if aPF && utils.SriovConfigured(dev.Address) { + // do not add this device in net device list + return + } + + if newDevice, err := resources.NewPciNetDevice(dev, rm.rFactory); err == nil { + rm.netDeviceList = append(rm.netDeviceList, newDevice) + } else { + glog.Errorf("discoverDevices() error adding new device: %q", err) + } + + } +} + +func (rm *resourceManager) accDevHandler(dev *ghw.PCIDevice) { + //Check if device is a physical function (PF) + aPF := utils.IsSriovPF(dev.Address) + devID, err := strconv.ParseInt(dev.Product.ID, 16, 64) + if err != nil { + glog.Warningf("discoverDevices(): unable to parse device ID for device %+v %q", dev, err) + return + } + //Check if device ID = 0b30, it means it is a RSU (Remote System Update) device and not accelerator + if devID != rsuDevice { + if aPF && utils.SriovConfigured(dev.Address) { + // If it is a PF and SRIOV is configured do not add to device list + return + } + + if newDevice, err := resources.NewPciNetDevice(dev, rm.rFactory); err == nil { + if newDevice.GetDriver() == "igb_uio" || newDevice.GetDriver() == "vfio-pci" { + rm.netDeviceList = append(rm.netDeviceList, newDevice) + } else { + glog.Infof("Excluding FEC device %-12s because it is not bound to userspace driver.", dev.Address) + } + } else { + glog.Errorf("discoverDevices() error adding new device: %q", err) + } + } else { + glog.Infof("Excluding device %-12s , it is a RSU device.", dev.Address) + } +}