hw/arm/smmu-common: (Temp) Fix address_space handling for nested case

commit: 6691a2f("hw/arm/smmu-common: Use sysmem for
get_address_space until !!s2_hwpti") mandates that for pass-through
devices we return sysmem address space until s2_hwpt is attached.

But this breaks the virtio-pci dev assignment especially when we assign
it without any pass-through vfio dev. This is a temporary(and ugly)
fix for that.

ToDo: Revisit this again.

Signed-off-by: Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>

hw/arm/smmu-common.c

@@ -620,7 +620,7 @@ static AddressSpace *smmu_find_add_as(PCIBus *bus, void *opaque, int devfn)
     SMMUPciBus *sbus = smmu_get_sbus(s, bus);
     SMMUDevice *sdev = smmu_get_sdev(s, sbus, bus, devfn);
 
-    if (s->nested && !s->s2_hwpt) {
+    if (s->nested && sdev->nested && !s->s2_hwpt) {
         return &sdev->as_sysmem;
     } else {
         return &sdev->as;
@@ -643,6 +643,16 @@ static int smmu_dev_set_iommu_device(PCIBus *bus, void *opaque, int devfn,
         return 0;
     }
 
+    /*
+     * VFIO makes a call early on without idev so that
+     * we can handle the address_sapce differently for
+     * nested cases untill dev is attached to s2_hwpt.
+     */
+    if (idev == NULL) {
+        sdev->nested = true;
+        return 0;
+    }
+
     if (QLIST_EMPTY(&s->devices_with_nesting)) {
         uint32_t s2_hwpt_id;
 

hw/vfio/pci.c

@@ -2990,6 +2990,17 @@ static void vfio_realize(PCIDevice *pdev, Error **errp)
         goto error;
     }
 
+    /*
+     * Inform IOMMU early on about the device, with idev as NULL.
+     * This helps to handle address_space differently for nested
+     * cases.
+     */
+    ret = pci_device_set_iommu_device(pdev, NULL, errp);
+    if (ret) {
+        error_prepend(errp, "Failed to set iommu_device: ");
+        goto error;
+    }
+
     if (!qemu_uuid_is_null(&vdev->vf_token)) {
         qemu_uuid_unparse(&vdev->vf_token, uuid);
         name = g_strdup_printf("%s vf_token=%s", vbasedev->name, uuid);
@@ -3107,11 +3118,15 @@ static void vfio_realize(PCIDevice *pdev, Error **errp)
 
     vfio_bars_register(vdev);
 
+    /* If iommufd is used, inform IOMMU about idev */
     if (vbasedev->iommufd) {
         ret = pci_device_set_iommu_device(pdev, &vbasedev->idev, errp);
-    } else {
-        ret = pci_device_set_iommu_device(pdev, 0, errp);
+        if (ret) {
+            error_prepend(errp, "Failed to set iommu_device: ");
+            goto out_teardown;
+        }
     }
+
     if (ret) {
         error_prepend(errp, "Failed to set iommu_device: ");
         goto out_teardown;

include/hw/arm/smmu-common.h

@@ -121,6 +121,7 @@ typedef struct SMMUDevice {
     IOMMUMemoryRegion  iommu;
     IOMMUFDDevice      *idev;
     SMMUHwpt           *hwpt;
+    bool               nested;
     AddressSpace       as;
     AddressSpace       as_sysmem;
     uint32_t           cfg_cache_hits;