s390x/pci: fix up IOMMU size

Present code uses @SiZe==UINT64_MAX to initialize IOMMU. It infers that it
can map any 64-bit IOVA whatsoever. But in fact, the largest DMA range for
each PCI Device on s390x is from ZPCI_SDMA_ADDR to ZPCI_EDMA_ADDR. The largest
value is returned from hardware, which is to indicate the largest range
hardware can support. But the real IOMMU size for specific PCI Device is
obtained once qemu intercepts mpcifc instruction that guest is requesting a
DMA range for that PCI Device. Therefore, before intercepting mpcifc instruction,
qemu cannot be aware of the size of IOMMU region that guest will use.

Moreover, iommu replay during device initialization for the whole region in
4k steps takes a very long time.

In conclusion, this patch intializes IOMMU region for each PCI Device when
intercept mpcifc instruction which is to register DMA range for the PCI Device.
And then, destroy IOMMU region when guest wants to deregister IOAT.

Signed-off-by: Yi Min Zhao <zyimin@linux.vnet.ibm.com>
Reviewed-by: Cornelia Huck <cornelia.huck@de.ibm.com>
Signed-off-by: Cornelia Huck <cornelia.huck@de.ibm.com>

hw/s390x/s390-pci-bus.c

@@ -308,7 +308,7 @@ static IOMMUTLBEntry s390_translate_iommu(MemoryRegion *iommu, hwaddr addr,
 {
     uint64_t pte;
     uint32_t flags;
-    S390PCIBusDevice *pbdev = container_of(iommu, S390PCIBusDevice, mr);
+    S390PCIBusDevice *pbdev = container_of(iommu, S390PCIBusDevice, iommu_mr);
     S390pciState *s;
     IOMMUTLBEntry ret = {
         .target_as = &address_space_memory,
@@ -454,14 +454,32 @@ static const MemoryRegionOps s390_msi_ctrl_ops = {
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
+void s390_pcihost_iommu_configure(S390PCIBusDevice *pbdev, bool enable)
+{
+    pbdev->configured = false;
+
+    if (enable) {
+        uint64_t size = pbdev->pal - pbdev->pba + 1;
+        memory_region_init_iommu(&pbdev->iommu_mr, OBJECT(&pbdev->mr),
+                                 &s390_iommu_ops, "iommu-s390", size);
+        memory_region_add_subregion(&pbdev->mr, pbdev->pba, &pbdev->iommu_mr);
+    } else {
+        memory_region_del_subregion(&pbdev->mr, &pbdev->iommu_mr);
+    }
+
+    pbdev->configured = true;
+}
+
 static void s390_pcihost_init_as(S390pciState *s)
 {
     int i;
+    S390PCIBusDevice *pbdev;
 
     for (i = 0; i < PCI_SLOT_MAX; i++) {
-        memory_region_init_iommu(&s->pbdev[i].mr, OBJECT(s),
-                                 &s390_iommu_ops, "iommu-s390", UINT64_MAX);
-        address_space_init(&s->pbdev[i].as, &s->pbdev[i].mr, "iommu-pci");
+        pbdev = &s->pbdev[i];
+        memory_region_init(&pbdev->mr, OBJECT(s),
+                           "iommu-root-s390", UINT64_MAX);
+        address_space_init(&pbdev->as, &pbdev->mr, "iommu-pci");
     }
 
     memory_region_init_io(&s->msix_notify_mr, OBJECT(s),

hw/s390x/s390-pci-bus.h

@@ -231,6 +231,7 @@ typedef struct S390PCIBusDevice {
     AdapterRoutes routes;
     AddressSpace as;
     MemoryRegion mr;
+    MemoryRegion iommu_mr;
 } S390PCIBusDevice;
 
 typedef struct S390pciState {
@@ -244,6 +245,7 @@ typedef struct S390pciState {
 int chsc_sei_nt2_get_event(void *res);
 int chsc_sei_nt2_have_event(void);
 void s390_pci_sclp_configure(int configure, SCCB *sccb);
+void s390_pcihost_iommu_configure(S390PCIBusDevice *pbdev, bool enable);
 S390PCIBusDevice *s390_pci_find_dev_by_idx(uint32_t idx);
 S390PCIBusDevice *s390_pci_find_dev_by_fh(uint32_t fh);
 S390PCIBusDevice *s390_pci_find_dev_by_fid(uint32_t fid);

hw/s390x/s390-pci-inst.c

@@ -528,7 +528,7 @@ int rpcit_service_call(S390CPU *cpu, uint8_t r1, uint8_t r2)
         goto out;
     }
 
-    mr = pci_device_iommu_address_space(pbdev->pdev)->root;
+    mr = &pbdev->iommu_mr;
     while (start < end) {
         entry = mr->iommu_ops->translate(mr, start, 0);
 
@@ -689,6 +689,9 @@ static int reg_ioat(CPUS390XState *env, S390PCIBusDevice *pbdev, ZpciFib fib)
     pbdev->pba = pba;
     pbdev->pal = pal;
     pbdev->g_iota = g_iota;
+
+    s390_pcihost_iommu_configure(pbdev, true);
+
     return 0;
 }
 
@@ -697,6 +700,8 @@ static void dereg_ioat(S390PCIBusDevice *pbdev)
     pbdev->pba = 0;
     pbdev->pal = 0;
     pbdev->g_iota = 0;
+
+    s390_pcihost_iommu_configure(pbdev, false);
 }
 
 int mpcifc_service_call(S390CPU *cpu, uint8_t r1, uint64_t fiba, uint8_t ar)