fix multiple pci device in one iommu group

fix pic device prefix fix unit test Signed-off-by: wiwen <shenggxhz@qq.com>
kubevirt · Nov 28, 2021 · e812e22 · e812e22
1 parent 9fd7bd4
commit e812e22
Show file tree

Hide file tree

Showing 3 changed files with 45 additions and 36 deletions.
diff --git a/pkg/virt-handler/device-manager/common.go b/pkg/virt-handler/device-manager/common.go
@@ -191,6 +191,13 @@ func (h *DeviceUtilsHandler) ReadMDEVAvailableInstances(mdevType string, parentI
 	return i, nil
 }
 
+func PciAddressToDevID(prefix string, pciAddr string) string {
+	varName := strings.ToLower(pciAddr)
+	varName = strings.Replace(varName, ":", "-", -1)
+	varName = strings.Replace(varName, ".", "-", -1)
+	return fmt.Sprintf("%s-%s", prefix, varName)
+}
+
 func initHandler() {
 	if Handler == nil {
 		Handler = &DeviceUtilsHandler{}

diff --git a/pkg/virt-handler/device-manager/pci_device.go b/pkg/virt-handler/device-manager/pci_device.go
@@ -42,6 +42,7 @@ const (
 	vfioMount           = "/dev/vfio/vfio"
 	pciBasePath         = "/sys/bus/pci/devices"
 	PCI_RESOURCE_PREFIX = "PCI_RESOURCE"
+	pciDevIDPrefix      = "pci"
 )
 
 type PCIDevice struct {
@@ -53,52 +54,53 @@ type PCIDevice struct {
 }
 
 type PCIDevicePlugin struct {
-	devs          []*pluginapi.Device
-	server        *grpc.Server
-	socketPath    string
-	stop          chan struct{}
-	health        chan string
-	devicePath    string
-	deviceName    string
-	resourceName  string
-	done          chan struct{}
-	deviceRoot    string
-	healthy       chan string
-	unhealthy     chan string
-	iommuToPCIMap map[string]string
-	initialized   bool
-	lock          *sync.Mutex
+	devs         []*pluginapi.Device
+	server       *grpc.Server
+	socketPath   string
+	stop         chan struct{}
+	health       chan string
+	devicePath   string
+	deviceName   string
+	resourceName string
+	done         chan struct{}
+	deviceRoot   string
+	healthy      chan string
+	unhealthy    chan string
+	devIDPCIMap  map[string]string
+	initialized  bool
+	lock         *sync.Mutex
 }
 
 func NewPCIDevicePlugin(pciDevices []*PCIDevice, resourceName string) *PCIDevicePlugin {
 	deviceIDStr := strings.Replace(pciDevices[0].pciID, ":", "-", -1)
 	serverSock := SocketPath(deviceIDStr)
-	iommuToPCIMap := make(map[string]string)
+	devIDPCIMap := make(map[string]string)
 
 	initHandler()
 
-	devs := constructDPIdevices(pciDevices, iommuToPCIMap)
+	devs := constructDPIdevices(pciDevices, devIDPCIMap)
 	dpi := &PCIDevicePlugin{
-		devs:          devs,
-		socketPath:    serverSock,
-		deviceName:    resourceName,
-		resourceName:  resourceName,
-		devicePath:    vfioDevicePath,
-		deviceRoot:    util.HostRootMount,
-		iommuToPCIMap: iommuToPCIMap,
-		healthy:       make(chan string),
-		unhealthy:     make(chan string),
-		initialized:   false,
-		lock:          &sync.Mutex{},
+		devs:         devs,
+		socketPath:   serverSock,
+		deviceName:   resourceName,
+		resourceName: resourceName,
+		devicePath:   vfioDevicePath,
+		deviceRoot:   util.HostRootMount,
+		devIDPCIMap:  devIDPCIMap,
+		healthy:      make(chan string),
+		unhealthy:    make(chan string),
+		initialized:  false,
+		lock:         &sync.Mutex{},
 	}
 	return dpi
 }
 
-func constructDPIdevices(pciDevices []*PCIDevice, iommuToPCIMap map[string]string) (devs []*pluginapi.Device) {
+func constructDPIdevices(pciDevices []*PCIDevice, devIDPCIMap map[string]string) (devs []*pluginapi.Device) {
 	for _, pciDevice := range pciDevices {
-		iommuToPCIMap[pciDevice.iommuGroup] = pciDevice.pciAddress
+		devID := PciAddressToDevID(pciDevIDPrefix, pciDevice.pciAddress)
+		devIDPCIMap[devID] = pciDevice.pciAddress
 		dpiDev := &pluginapi.Device{
-			ID:     string(pciDevice.iommuGroup),
+			ID:     devID,
 			Health: pluginapi.Healthy,
 		}
 		if pciDevice.numaNode >= 0 {
@@ -222,8 +224,8 @@ func (dpi *PCIDevicePlugin) Allocate(_ context.Context, r *pluginapi.AllocateReq
 	for _, request := range r.ContainerRequests {
 		deviceSpecs := make([]*pluginapi.DeviceSpec, 0)
 		for _, devID := range request.DevicesIDs {
-			// translate device's iommu group to its pci address
-			devPCIAddress, exist := dpi.iommuToPCIMap[devID]
+			// translate device's  to its pci address
+			devPCIAddress, exist := dpi.devIDPCIMap[devID]
 			if !exist {
 				continue
 			}

diff --git a/pkg/virt-handler/device-manager/pci_device_test.go b/pkg/virt-handler/device-manager/pci_device_test.go
@@ -94,7 +94,7 @@ pciHostDevices:
 	})
 
 	It("Should validate DPI devices", func() {
-		iommuToPCIMap := make(map[string]string)
+		devIDPCIMap := make(map[string]string)
 		supportedPCIDeviceMap := make(map[string]string)
 		for _, pciDev := range fakePermittedHostDevices.PciHostDevices {
 			// do not add a device plugin for this resource if it's being provided via an external device plugin
@@ -105,8 +105,8 @@ pciHostDevices:
 		// discoverPermittedHostPCIDevices() will walk real PCI devices wherever the tests are running
 		// It's assumed here that it will find a PCI device at 0000:00:00.0
 		pciDevices := discoverPermittedHostPCIDevices(supportedPCIDeviceMap)
-		devs := constructDPIdevices(pciDevices[fakeID], iommuToPCIMap)
-		Expect(devs[0].ID).To(Equal(fakeIommuGroup))
+		devs := constructDPIdevices(pciDevices[fakeID], devIDPCIMap)
+		Expect(devs[0].ID).To(Equal(PciAddressToDevID(pciDevIDPrefix, fakeAddress)))
 		Expect(devs[0].Topology.Nodes[0].ID).To(Equal(int64(fakeNumaNode)))
 	})
 	It("Should update the device list according to the configmap", func() {