device: fix detach function to remove hot plugged devices

use PCI bridges to remove devices, the bridge will update its list
of devices and addressese to be able to reuse empty slots

Signed-off-by: Julio Montes <julio.montes@intel.com>

container.go

@@ -867,7 +867,7 @@ func (c *Container) attachDevices() error {
 
 func (c *Container) detachDevices() error {
 	for _, device := range c.devices {
-		if err := device.detach(c.pod.hypervisor); err != nil {
+		if err := device.detach(c.pod.hypervisor, c); err != nil {
 			return err
 		}
 	}

device.go

@@ -60,7 +60,7 @@ const (
 // Device is the virtcontainers device interface.
 type Device interface {
 	attach(hypervisor, *Container) error
-	detach(hypervisor) error
+	detach(hypervisor, *Container) error
 	deviceType() string
 }
 
@@ -153,7 +153,7 @@ func (device *VFIODevice) attach(h hypervisor, c *Container) error {
 	return nil
 }
 
-func (device *VFIODevice) detach(h hypervisor) error {
+func (device *VFIODevice) detach(h hypervisor, c *Container) error {
 	return nil
 }
 
@@ -252,19 +252,26 @@ func (device *BlockDevice) attach(h hypervisor, c *Container) (err error) {
 	return nil
 }
 
-func (device BlockDevice) detach(h hypervisor) error {
+func (device BlockDevice) detach(h hypervisor, c *Container) error {
 	if device.DeviceInfo.Hotplugged {
 		deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Unplugging block device")
 
 		drive := Drive{
 			ID: makeBlockDevIDForHypervisor(device.DeviceInfo.ID),
 		}
 
-		if err := h.hotplugRemoveDevice(drive, blockDev); err != nil {
-			deviceLogger().WithError(err).Error("Failed to unplug block device")
-			return err
+		var err error
+		for _, b := range c.pod.config.VMConfig.Bridges {
+			err = b.hotplugRemovePCIDevice(h, drive, blockDev)
+			if err == nil {
+				// devices was hot unplugged correctly
+				break
+			}
 		}
 
+		if err != nil {
+			return err
+		}
 	}
 	return nil
 }
@@ -290,7 +297,7 @@ func (device *GenericDevice) attach(h hypervisor, c *Container) error {
 	return nil
 }
 
-func (device *GenericDevice) detach(h hypervisor) error {
+func (device *GenericDevice) detach(h hypervisor, c *Container) error {
 	return nil
 }