From 49d73c7771e3f6054804f6cfa80b4e320111662d Mon Sep 17 00:00:00 2001 From: Alexey Kardashevskiy Date: Tue, 31 Mar 2020 12:09:48 +1100 Subject: [PATCH] powerpc/dma: Call indirect dma_ops when persistent memory present Unlike normal memory ("memory" compatible type in the FDT), the persistent memory ("ibm,pmemory" in the FDT) can be mapped anywhere in the guest physical space and it can be used for DMA. In order to maintain 1:1 mapping via the huge DMA window, we need to know the maximum physical address at the time of the window setup. So far we've been looking at "memory" nodes but "ibm,pmemory" does not have fixed addresses and the persistent memory may be mapped afterwards. When persistent memory present, this clears the dma_ops_bypass flag to tell the generic code that indirect dma_ops call is needed. This lets the platform code check the DMA boundaries and call direct DMA if possible or IOMMU API otherwise. If dma_ops_bypass is not set, this goes to IOMMU DMA unconditionally as we cannot easily predict where the memory for DMA is going to be allocated from (can be ibm,pmemory as well). This limits IOMMU bypass by max_pfn so this affects powernv as well. This should not change the existing behaviour in absence of persistent memory. Signed-off-by: Alexey Kardashevskiy --- This is based on http://git.infradead.org/users/hch/misc.git/shortlog/refs/heads/dma-bypass.3 --- arch/powerpc/kernel/dma-iommu.c | 74 ++++++++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 2 deletions(-) diff --git a/arch/powerpc/kernel/dma-iommu.c b/arch/powerpc/kernel/dma-iommu.c index 569fecd7b5b234..96eb84a3832c19 100644 --- a/arch/powerpc/kernel/dma-iommu.c +++ b/arch/powerpc/kernel/dma-iommu.c @@ -10,6 +10,31 @@ #include #include +static bool can_map_direct(struct device *dev, phys_addr_t addr, size_t size) +{ + dma_addr_t max_dma, dma_handle; + + if (!dev->archdata.dma_offset) + return false; + + max_dma = phys_to_dma(dev, (max_pfn - 1) << PAGE_SHIFT); + dma_handle = phys_to_dma(dev, addr + size); + + return max_dma >= dma_handle; +} + +static bool dma_handle_direct(struct device *dev, dma_addr_t dma_handle) +{ + dma_addr_t max_dma; + + if (!dev->archdata.dma_offset) + return false; + + max_dma = phys_to_dma(dev, (max_pfn - 1) << PAGE_SHIFT); + + return max_dma >= dma_handle; +} + /* * Generic iommu implementation */ @@ -44,6 +69,11 @@ static dma_addr_t dma_iommu_map_page(struct device *dev, struct page *page, enum dma_data_direction direction, unsigned long attrs) { + if (can_map_direct(dev, (phys_addr_t) page_to_phys(page), + offset + size)) + return dma_direct_map_page(dev, page, offset, size, direction, + attrs); + return iommu_map_page(dev, get_iommu_table_base(dev), page, offset, size, dma_get_mask(dev), direction, attrs); } @@ -53,6 +83,10 @@ static void dma_iommu_unmap_page(struct device *dev, dma_addr_t dma_handle, size_t size, enum dma_data_direction direction, unsigned long attrs) { + if (dma_handle_direct(dev, dma_handle)) { + dma_direct_unmap_page(dev, dma_handle, size, direction, attrs); + return; + } iommu_unmap_page(get_iommu_table_base(dev), dma_handle, size, direction, attrs); } @@ -62,6 +96,18 @@ static int dma_iommu_map_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction direction, unsigned long attrs) { + struct scatterlist *s; + bool direct = true; + int i; + + for_each_sg(sglist, s, nelems, i) { + direct = can_map_direct(dev, sg_phys(s), s->offset + s->length); + if (!direct) + break; + } + if (direct) + return dma_direct_map_sg(dev, sglist, nelems, direction, attrs); + return ppc_iommu_map_sg(dev, get_iommu_table_base(dev), sglist, nelems, dma_get_mask(dev), direction, attrs); } @@ -70,6 +116,20 @@ static void dma_iommu_unmap_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction direction, unsigned long attrs) { + struct scatterlist *s; + bool direct = true; + int i; + + for_each_sg(sglist, s, nelems, i) { + direct = dma_handle_direct(dev, s->dma_address); + if (!direct) + break; + } + if (direct) { + dma_direct_unmap_sg(dev, sglist, nelems, direction, attrs); + return; + } + ppc_iommu_unmap_sg(get_iommu_table_base(dev), sglist, nelems, direction, attrs); } @@ -90,8 +150,18 @@ int dma_iommu_dma_supported(struct device *dev, u64 mask) struct iommu_table *tbl = get_iommu_table_base(dev); if (dev_is_pci(dev) && dma_iommu_bypass_supported(dev, mask)) { - dev->dma_ops_bypass = true; - dev_dbg(dev, "iommu: 64-bit OK, using fixed ops\n"); + /* + * The "ibm,pmemory" can appear anywhere in the address space + * so may not be able to map it via the 1:1 window, force + * generic DMA to go via dma_iommu_ops. + */ + dev->dma_ops_bypass = + of_find_node_by_type(NULL, "ibm,pmemory") == NULL; + + if (!dev->dma_ops_bypass) + dev_warn(dev, "iommu: 64-bit OK but using default ops\n"); + else + dev_dbg(dev, "iommu: 64-bit OK, using fixed ops\n"); return 1; }