powerpc/dma: Call indirect dma_ops when persistent memory present

Unlike normal memory ("memory" compatible type in the FDT),
the persistent memory ("ibm,pmemory" in the FDT) can be mapped anywhere
in the guest physical space and it can be used for DMA.

In order to maintain 1:1 mapping via the huge DMA window, we need to
know the maximum physical address at the time of the window setup.
So far we've been looking at "memory" nodes but "ibm,pmemory" does not
have fixed addresses and the persistent memory may be mapped afterwards.

When persistent memory present, this clears the dma_ops_bypass flag to
tell the generic code that indirect dma_ops call is needed. This lets
the platform code check the DMA boundaries and call direct DMA if
possible or IOMMU API otherwise.

If dma_ops_bypass is not set, this goes to IOMMU DMA unconditionally as
we cannot easily predict where the memory for DMA is going to be
allocated from (can be ibm,pmemory as well).

This limits IOMMU bypass by max_pfn so this affects powernv as well.

This should not change the existing behaviour in absence of persistent
memory.

Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
---

This is based on
http://git.infradead.org/users/hch/misc.git/shortlog/refs/heads/dma-bypass.3

arch/powerpc/kernel/dma-iommu.c

@@ -10,6 +10,31 @@
 #include <linux/pci.h>
 #include <asm/iommu.h>
 
+static bool can_map_direct(struct device *dev, phys_addr_t addr, size_t size)
+{
+	dma_addr_t max_dma, dma_handle;
+
+	if (!dev->archdata.dma_offset)
+		return false;
+
+	max_dma = phys_to_dma(dev, (max_pfn - 1) << PAGE_SHIFT);
+	dma_handle = phys_to_dma(dev, addr + size);
+
+	return max_dma >= dma_handle;
+}
+
+static bool dma_handle_direct(struct device *dev, dma_addr_t dma_handle)
+{
+	dma_addr_t max_dma;
+
+	if (!dev->archdata.dma_offset)
+		return false;
+
+	max_dma = phys_to_dma(dev, (max_pfn - 1) << PAGE_SHIFT);
+
+	return max_dma >= dma_handle;
+}
+
 /*
  * Generic iommu implementation
  */
@@ -44,6 +69,11 @@ static dma_addr_t dma_iommu_map_page(struct device *dev, struct page *page,
 				     enum dma_data_direction direction,
 				     unsigned long attrs)
 {
+	if (can_map_direct(dev, (phys_addr_t) page_to_phys(page),
+			   offset + size))
+		return dma_direct_map_page(dev, page, offset, size, direction,
+					   attrs);
+
 	return iommu_map_page(dev, get_iommu_table_base(dev), page, offset,
 			      size, dma_get_mask(dev), direction, attrs);
 }
@@ -53,6 +83,10 @@ static void dma_iommu_unmap_page(struct device *dev, dma_addr_t dma_handle,
 				 size_t size, enum dma_data_direction direction,
 				 unsigned long attrs)
 {
+	if (dma_handle_direct(dev, dma_handle)) {
+		dma_direct_unmap_page(dev, dma_handle, size, direction, attrs);
+		return;
+	}
 	iommu_unmap_page(get_iommu_table_base(dev), dma_handle, size, direction,
 			 attrs);
 }
@@ -62,6 +96,18 @@ static int dma_iommu_map_sg(struct device *dev, struct scatterlist *sglist,
 			    int nelems, enum dma_data_direction direction,
 			    unsigned long attrs)
 {
+	struct scatterlist *s;
+	bool direct = true;
+	int i;
+
+	for_each_sg(sglist, s, nelems, i) {
+		direct = can_map_direct(dev, sg_phys(s), s->offset + s->length);
+		if (!direct)
+			break;
+	}
+	if (direct)
+		return dma_direct_map_sg(dev, sglist, nelems, direction, attrs);
+
 	return ppc_iommu_map_sg(dev, get_iommu_table_base(dev), sglist, nelems,
 				dma_get_mask(dev), direction, attrs);
 }
@@ -70,6 +116,20 @@ static void dma_iommu_unmap_sg(struct device *dev, struct scatterlist *sglist,
 		int nelems, enum dma_data_direction direction,
 		unsigned long attrs)
 {
+	struct scatterlist *s;
+	bool direct = true;
+	int i;
+
+	for_each_sg(sglist, s, nelems, i) {
+		direct = dma_handle_direct(dev, s->dma_address);
+		if (!direct)
+			break;
+	}
+	if (direct) {
+		dma_direct_unmap_sg(dev, sglist, nelems, direction, attrs);
+		return;
+	}
+
 	ppc_iommu_unmap_sg(get_iommu_table_base(dev), sglist, nelems,
 			   direction, attrs);
 }
@@ -90,8 +150,18 @@ int dma_iommu_dma_supported(struct device *dev, u64 mask)
 	struct iommu_table *tbl = get_iommu_table_base(dev);
 
 	if (dev_is_pci(dev) && dma_iommu_bypass_supported(dev, mask)) {
-		dev->dma_ops_bypass = true;
-		dev_dbg(dev, "iommu: 64-bit OK, using fixed ops\n");
+		/*
+		 * The "ibm,pmemory" can appear anywhere in the address space
+		 * so may not be able to map it via the 1:1 window, force
+		 * generic DMA to go via dma_iommu_ops.
+		 */
+		dev->dma_ops_bypass =
+			of_find_node_by_type(NULL, "ibm,pmemory") == NULL;
+
+		if (!dev->dma_ops_bypass)
+			dev_warn(dev, "iommu: 64-bit OK but using default ops\n");
+		else
+			dev_dbg(dev, "iommu: 64-bit OK, using fixed ops\n");
 		return 1;
 	}