PCI: hv: Add a per-bus mutex state_lock

Bugzilla: https://bugzilla.redhat.com/show_bug.cgi?id=2182619

commit 067d6ec
Author: Dexuan Cui <decui@microsoft.com>
Date:   Wed Jun 14 21:44:51 2023 -0700

    PCI: hv: Add a per-bus mutex state_lock

    In the case of fast device addition/removal, it's possible that
    hv_eject_device_work() can start to run before create_root_hv_pci_bus()
    starts to run; as a result, the pci_get_domain_bus_and_slot() in
    hv_eject_device_work() can return a 'pdev' of NULL, and
    hv_eject_device_work() can remove the 'hpdev', and immediately send a
    message PCI_EJECTION_COMPLETE to the host, and the host immediately
    unassigns the PCI device from the guest; meanwhile,
    create_root_hv_pci_bus() and the PCI device driver can be probing the
    dead PCI device and reporting timeout errors.

    Fix the issue by adding a per-bus mutex 'state_lock' and grabbing the
    mutex before powering on the PCI bus in hv_pci_enter_d0(): when
    hv_eject_device_work() starts to run, it's able to find the 'pdev' and call
    pci_stop_and_remove_bus_device(pdev): if the PCI device driver has
    loaded, the PCI device driver's probe() function is already called in
    create_root_hv_pci_bus() -> pci_bus_add_devices(), and now
    hv_eject_device_work() -> pci_stop_and_remove_bus_device() is able
    to call the PCI device driver's remove() function and remove the device
    reliably; if the PCI device driver hasn't loaded yet, the function call
    hv_eject_device_work() -> pci_stop_and_remove_bus_device() is able to
    remove the PCI device reliably and the PCI device driver's probe()
    function won't be called; if the PCI device driver's probe() is already
    running (e.g., systemd-udev is loading the PCI device driver), it must
    be holding the per-device lock, and after the probe() finishes and releases
    the lock, hv_eject_device_work() -> pci_stop_and_remove_bus_device() is
    able to proceed to remove the device reliably.

    Fixes: 4daace0 ("PCI: hv: Add paravirtual PCI front-end for Microsoft Hyper-V VMs")
    Signed-off-by: Dexuan Cui <decui@microsoft.com>
    Reviewed-by: Michael Kelley <mikelley@microsoft.com>
    Acked-by: Lorenzo Pieralisi <lpieralisi@kernel.org>
    Cc: stable@vger.kernel.org
    Link: https://lore.kernel.org/r/20230615044451.5580-6-decui@microsoft.com
    Signed-off-by: Wei Liu <wei.liu@kernel.org>

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>

Author: vittyvk

drivers/pci/controller/pci-hyperv.c

@@ -489,7 +489,10 @@ struct hv_pcibus_device {
 	struct fwnode_handle *fwnode;
 	/* Protocol version negotiated with the host */
 	enum pci_protocol_version_t protocol_version;
+
+	struct mutex state_lock;
 	enum hv_pcibus_state state;
+
 	struct hv_device *hdev;
 	resource_size_t low_mmio_space;
 	resource_size_t high_mmio_space;
@@ -2512,6 +2515,8 @@ static void pci_devices_present_work(struct work_struct *work)
 	if (!dr)
 		return;
 
+	mutex_lock(&hbus->state_lock);
+
 	/* First, mark all existing children as reported missing. */
 	spin_lock_irqsave(&hbus->device_list_lock, flags);
 	list_for_each_entry(hpdev, &hbus->children, list_entry) {
@@ -2593,6 +2598,8 @@ static void pci_devices_present_work(struct work_struct *work)
 		break;
 	}
 
+	mutex_unlock(&hbus->state_lock);
+
 	kfree(dr);
 }
 
@@ -2741,6 +2748,8 @@ static void hv_eject_device_work(struct work_struct *work)
 	hpdev = container_of(work, struct hv_pci_dev, wrk);
 	hbus = hpdev->hbus;
 
+	mutex_lock(&hbus->state_lock);
+
 	/*
 	 * Ejection can come before or after the PCI bus has been set up, so
 	 * attempt to find it and tear down the bus state, if it exists.  This
@@ -2777,6 +2786,8 @@ static void hv_eject_device_work(struct work_struct *work)
 	put_pcichild(hpdev);
 	put_pcichild(hpdev);
 	/* hpdev has been freed. Do not use it any more. */
+
+	mutex_unlock(&hbus->state_lock);
 }
 
 /**
@@ -3567,6 +3578,7 @@ static int hv_pci_probe(struct hv_device *hdev,
 		return -ENOMEM;
 
 	hbus->bridge = bridge;
+	mutex_init(&hbus->state_lock);
 	hbus->state = hv_pcibus_init;
 	hbus->wslot_res_allocated = -1;
 
@@ -3675,9 +3687,11 @@ static int hv_pci_probe(struct hv_device *hdev,
 	if (ret)
 		goto free_irq_domain;
 
+	mutex_lock(&hbus->state_lock);
+
 	ret = hv_pci_enter_d0(hdev);
 	if (ret)
-		goto free_irq_domain;
+		goto release_state_lock;
 
 	ret = hv_pci_allocate_bridge_windows(hbus);
 	if (ret)
@@ -3695,12 +3709,15 @@ static int hv_pci_probe(struct hv_device *hdev,
 	if (ret)
 		goto free_windows;
 
+	mutex_unlock(&hbus->state_lock);
 	return 0;
 
 free_windows:
 	hv_pci_free_bridge_windows(hbus);
 exit_d0:
 	(void) hv_pci_bus_exit(hdev, true);
+release_state_lock:
+	mutex_unlock(&hbus->state_lock);
 free_irq_domain:
 	irq_domain_remove(hbus->irq_domain);
 free_fwnode:
@@ -3950,20 +3967,26 @@ static int hv_pci_resume(struct hv_device *hdev)
 	if (ret)
 		goto out;
 
+	mutex_lock(&hbus->state_lock);
+
 	ret = hv_pci_enter_d0(hdev);
 	if (ret)
-		goto out;
+		goto release_state_lock;
 
 	ret = hv_send_resources_allocated(hdev);
 	if (ret)
-		goto out;
+		goto release_state_lock;
 
 	prepopulate_bars(hbus);
 
 	hv_pci_restore_msi_state(hbus);
 
 	hbus->state = hv_pcibus_installed;
+	mutex_unlock(&hbus->state_lock);
 	return 0;
+
+release_state_lock:
+	mutex_unlock(&hbus->state_lock);
 out:
 	vmbus_close(hdev->channel);
 	return ret;