Add support for unmapped and userspace buffers in BUSDMA

Author: strejda

sys/arm/arm/busdma_machdep-v6.c

@@ -61,7 +61,7 @@ __FBSDID("$FreeBSD$");
 
 #include <machine/atomic.h>
 #include <machine/bus.h>
-#include <machine/cpufunc.h>
+#include <machine/cpu-v6.h>
 #include <machine/md_var.h>
 
 #define MAX_BPAGES 64
@@ -110,8 +110,8 @@ struct bounce_page {
 };
 
 struct sync_list {
-	vm_offset_t	vaddr;		/* kva of bounce buffer */
-	bus_addr_t	busaddr;	/* Physical address */
+	vm_offset_t	vaddr;		/* kva of client data */
+	bus_addr_t	busaddr;	/* client physical address */
 	bus_size_t	datacount;	/* client data count */
 };
 
@@ -625,7 +625,8 @@ bus_dma_tag_destroy(bus_dma_tag_t dmat)
 	return (error);
 }
 
-static int allocate_bz_and_pages(bus_dma_tag_t dmat, bus_dmamap_t mapp)
+static int
+allocate_bz_and_pages(bus_dma_tag_t dmat, bus_dmamap_t mapp)
 {
 	struct bounce_zone *bz;
 	int maxpages;
@@ -1043,6 +1044,7 @@ _bus_dmamap_load_phys(bus_dma_tag_t dmat,
 {
 	bus_addr_t curaddr;
 	bus_size_t sgsize;
+	struct sync_list *sl;
 	int error;
 
 	if (segs == NULL)
@@ -1051,7 +1053,7 @@ _bus_dmamap_load_phys(bus_dma_tag_t dmat,
 	counter_u64_add(maploads_total, 1);
 	counter_u64_add(maploads_physmem, 1);
 
-	if (might_bounce(dmat, map, buflen, buflen)) {
+	if (might_bounce(dmat, map, (bus_addr_t)buf, buflen)) {
 		_bus_dmamap_count_phys(dmat, map, buf, buflen, flags);
 		if (map->pagesneeded != 0) {
 			counter_u64_add(maploads_bounced, 1);
@@ -1069,6 +1071,18 @@ _bus_dmamap_load_phys(bus_dma_tag_t dmat,
 			sgsize = MIN(sgsize, PAGE_SIZE);
 			curaddr = add_bounce_page(dmat, map, 0, curaddr,
 						  sgsize);
+		} else {
+			sl = &map->slist[map->sync_count - 1];
+			if (map->sync_count == 0 ||
+			    curaddr != sl->busaddr + sl->datacount) {
+				if (++map->sync_count > dmat->nsegments)
+					goto cleanup;
+				sl++;
+				sl->vaddr = 0;
+				sl->datacount = sgsize;
+				sl->busaddr = curaddr;
+			} else
+				sl->datacount += sgsize;
 		}
 		sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
 		    segp);
@@ -1078,6 +1092,7 @@ _bus_dmamap_load_phys(bus_dma_tag_t dmat,
 		buflen -= sgsize;
 	}
 
+cleanup:
 	/*
 	 * Did we fit?
 	 */
@@ -1171,9 +1186,7 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dmat,
 		} else {
 			sl = &map->slist[map->sync_count - 1];
 			if (map->sync_count == 0 ||
-#ifdef ARM_L2_PIPT
 			    curaddr != sl->busaddr + sl->datacount ||
-#endif
 			    vaddr != sl->vaddr + sl->datacount) {
 				if (++map->sync_count > dmat->nsegments)
 					goto cleanup;
@@ -1251,58 +1264,21 @@ _bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
 	map->flags &= ~DMAMAP_MBUF;
 }
 
-#ifdef notyetbounceuser
-/* If busdma uses user pages, then the interrupt handler could
- * be use the kernel vm mapping. Both bounce pages and sync list
- * do not cross page boundaries.
- * Below is a rough sequence that a person would do to fix the
- * user page reference in the kernel vmspace. This would be
- * done in the dma post routine.
- */
-void
-_bus_dmamap_fix_user(vm_offset_t buf, bus_size_t len,
-			pmap_t pmap, int op)
+static void
+dma_dcache_ops(vm_paddr_t pa, vm_size_t size, int ops)
 {
-	bus_size_t sgsize;
-	bus_addr_t curaddr;
-	vm_offset_t va;
-
-	/*
-	 * each synclist entry is contained within a single page.
-	 * this would be needed if BUS_DMASYNC_POSTxxxx was implemented
-	 */
-	curaddr = pmap_extract(pmap, buf);
-	va = pmap_dma_map(curaddr);
-	switch (op) {
-	case SYNC_USER_INV:
-		cpu_dcache_wb_range(va, sgsize);
-		break;
-
-	case SYNC_USER_COPYTO:
-		bcopy((void *)va, (void *)bounce, sgsize);
-		break;
-
-	case SYNC_USER_COPYFROM:
-		bcopy((void *) bounce, (void *)va, sgsize);
-		break;
-
-	default:
-		break;
+	uint32_t len, offset;
+	vm_page_t m;
+
+	offset = pa & PAGE_MASK;
+	for ( ; size != 0; size -= len, pa += len, offset = 0) {
+		len = min(PAGE_SIZE - offset, size);
+		m = PHYS_TO_VM_PAGE(pa);
+		KASSERT(m != NULL, ("%s: vm_page_t is null for %#x",
+		  __func__, pa));
+		pmap_dma_dcache_ops(pa, len, m->md.pat_mode, ops);
 	}
-
-	pmap_dma_unmap(va);
 }
-#endif
-
-#ifdef ARM_L2_PIPT
-#define l2cache_wb_range(va, pa, size) cpu_l2cache_wb_range(pa, size)
-#define l2cache_wbinv_range(va, pa, size) cpu_l2cache_wbinv_range(pa, size)
-#define l2cache_inv_range(va, pa, size) cpu_l2cache_inv_range(pa, size)
-#else
-#define l2cache_wb_range(va, pa, size) cpu_l2cache_wb_range(va, size)
-#define l2cache_wbinv_range(va, pa, size) cpu_l2cache_wbinv_range(va, size)
-#define l2cache_inv_range(va, pa, size) cpu_l2cache_inv_range(va, size)
-#endif
 
 void
 _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
@@ -1316,9 +1292,6 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
 	 * we're able to test direct userland dma, panic on a map mismatch.
 	 */
 	if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
-		if (!pmap_dmap_iscurrent(map->pmap))
-			panic("_bus_dmamap_sync: wrong user map for bounce sync.");
-
 		CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x "
 		    "performing bounce", __func__, dmat, dmat->flags, op);
 
@@ -1328,19 +1301,17 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
 		 */
 		if (op & BUS_DMASYNC_PREWRITE) {
 			while (bpage != NULL) {
-				if (bpage->datavaddr != 0)
+				if (bpage->datavaddr != 0 &&
+				    pmap_dmap_iscurrent(map->pmap))
 					bcopy((void *)bpage->datavaddr,
 					    (void *)bpage->vaddr,
 					    bpage->datacount);
 				else
 					physcopyout(bpage->dataaddr,
 					    (void *)bpage->vaddr,
 					    bpage->datacount);
-				cpu_dcache_wb_range((vm_offset_t)bpage->vaddr,
-				    bpage->datacount);
-				l2cache_wb_range((vm_offset_t)bpage->vaddr,
-				    (vm_offset_t)bpage->busaddr,
-				    bpage->datacount);
+				dcache_wb_poc(bpage->vaddr,
+				    (vm_paddr_t)bpage->busaddr, bpage->datacount);
 				bpage = STAILQ_NEXT(bpage, links);
 			}
 			dmat->bounce_zone->total_bounced++;
@@ -1360,11 +1331,8 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
 		if ((op & BUS_DMASYNC_PREREAD) && !(op & BUS_DMASYNC_PREWRITE)) {
 			bpage = STAILQ_FIRST(&map->bpages);
 			while (bpage != NULL) {
-				cpu_dcache_inv_range((vm_offset_t)bpage->vaddr,
-				    bpage->datacount);
-				l2cache_inv_range((vm_offset_t)bpage->vaddr,
-				    (vm_offset_t)bpage->busaddr,
-				    bpage->datacount);
+				dcache_dma_preread(bpage->vaddr,
+				    (vm_paddr_t)bpage->busaddr, bpage->datacount);
 				bpage = STAILQ_NEXT(bpage, links);
 			}
 		}
@@ -1380,23 +1348,10 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
 		 */
 		if (op & BUS_DMASYNC_POSTREAD) {
 			while (bpage != NULL) {
-				vm_offset_t startv;
-				vm_paddr_t startp;
-				int len;
-
-				startv = bpage->vaddr &~ arm_dcache_align_mask;
-				startp = bpage->busaddr &~ arm_dcache_align_mask;
-				len = bpage->datacount;
-
-				if (startv != bpage->vaddr)
-					len += bpage->vaddr & arm_dcache_align_mask;
-				if (len & arm_dcache_align_mask)
-					len = (len -
-					    (len & arm_dcache_align_mask)) +
-					    arm_dcache_align;
-				l2cache_inv_range(startv, startp, len);
-				cpu_dcache_inv_range(startv, len);
-				if (bpage->datavaddr != 0)
+				dcache_inv_poc(bpage->vaddr,
+				    (vm_paddr_t)bpage->busaddr, bpage->datacount);
+				if (bpage->datavaddr != 0 &&
+				    pmap_dmap_iscurrent(map->pmap))
 					bcopy((void *)bpage->vaddr,
 					    (void *)bpage->datavaddr,
 					    bpage->datacount);
@@ -1434,9 +1389,6 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
 	 * outer-to-inner for POSTREAD invalidation is not a mistake.
 	 */
 	if (map->sync_count != 0) {
-		if (!pmap_dmap_iscurrent(map->pmap))
-			panic("_bus_dmamap_sync: wrong user map for sync.");
-
 		sl = &map->slist[0];
 		end = &map->slist[map->sync_count];
 		CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x "
@@ -1446,9 +1398,14 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
 		case BUS_DMASYNC_PREWRITE:
 		case BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD:
 			while (sl != end) {
-				cpu_dcache_wb_range(sl->vaddr, sl->datacount);
-				l2cache_wb_range(sl->vaddr, sl->busaddr,
-				    sl->datacount);
+				if (sl->vaddr != 0 &&
+				    pmap_dmap_iscurrent(map->pmap))
+					dcache_wb_poc(sl->vaddr,
+					    (vm_paddr_t)sl->busaddr,
+					    sl->datacount);
+				else
+					dma_dcache_ops((vm_paddr_t)sl->busaddr,
+					    sl->datacount, DDO_WB_POC);
 				sl++;
 			}
 			break;
@@ -1465,16 +1422,18 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
 			 * they are not aligned to a cacheline.
 			 */
 			while (sl != end) {
-				if (sl->vaddr & arm_dcache_align_mask) {
+				if (sl->busaddr & cpuinfo.dcache_line_mask) {
 					KASSERT(map->flags & DMAMAP_MBUF,
 					    ("unaligned buffer is not an mbuf"));
-					cpu_dcache_wb_range(sl->vaddr, 1);
-					l2cache_wb_range(sl->vaddr,
-					    sl->busaddr, 1);
 				}
-				cpu_dcache_inv_range(sl->vaddr, sl->datacount);
-				l2cache_inv_range(sl->vaddr, sl->busaddr,
-				    sl->datacount);
+				if (sl->vaddr != 0 &&
+				    pmap_dmap_iscurrent(map->pmap))
+					dcache_dma_preread_safe(sl->vaddr,
+					    (vm_paddr_t)sl->busaddr,
+					    sl->datacount);
+				else
+					dma_dcache_ops((vm_paddr_t)sl->busaddr,
+					    sl->datacount, DDO_PREREAD_SAFE);
 				sl++;
 			}
 			break;
@@ -1485,9 +1444,14 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
 		case BUS_DMASYNC_POSTREAD:
 		case BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE:
 			while (sl != end) {
-				l2cache_inv_range(sl->vaddr, sl->busaddr,
-				    sl->datacount);
-				cpu_dcache_inv_range(sl->vaddr, sl->datacount);
+				if (sl->vaddr != 0 &&
+				    pmap_dmap_iscurrent(map->pmap))
+					dcache_inv_poc(sl->vaddr,
+					    (vm_paddr_t)sl->busaddr,
+					    sl->datacount);
+				else
+					dma_dcache_ops((vm_paddr_t)sl->busaddr,
+					    sl->datacount, DDO_INV_POC);
 				sl++;
 			}
 			break;

sys/arm/arm/pmap-v6-new.c

@@ -6076,6 +6076,48 @@ pmap_sync_icache(pmap_t pmap, vm_offset_t va, vm_size_t size)
 	icache_inv_all();
 }
 
+/*
+ *  Do DMA ops on data caches range by physical address.
+ *  The range must be within a single page.
+ */
+void
+pmap_dma_dcache_ops(vm_paddr_t pa, vm_size_t size, vm_memattr_t ma, int ops)
+{
+	struct sysmaps *sysmaps;
+	vm_offset_t va;
+
+	KASSERT(((pa & PAGE_MASK) + size) <= PAGE_SIZE,
+	    ("%s: not on single page", __func__));
+
+	sched_pin();
+	sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)];
+	mtx_lock(&sysmaps->lock);
+	if (*sysmaps->CMAP3)
+		panic("%s: CMAP3 busy", __func__);
+	pte2_store(sysmaps->CMAP3, PTE2_KERN_NG(pa, PTE2_AP_KRW, ma));
+	tlb_flush_local((vm_offset_t)sysmaps->CADDR3);
+	va = (vm_offset_t)sysmaps->CADDR3 + (pa & PAGE_MASK);
+	switch (ops) {
+	case DDO_WB_POC:
+		dcache_wb_poc(va, pa, size);
+		break;
+	case DDO_INV_POC:
+		dcache_inv_poc(va, pa, size);
+		break;
+	case DDO_PREREAD:
+		dcache_dma_preread(va, pa, size);
+		break;
+	case DDO_PREREAD_SAFE:
+		dcache_dma_preread_safe(va, pa, size);
+		break;
+	default:
+		panic("%s: unknown ops %d", __func__, ops);
+	}
+	pte2_clear(sysmaps->CMAP3);
+	sched_unpin();
+	mtx_unlock(&sysmaps->lock);
+}
+
 /*
  *  The implementation of pmap_fault() uses IN_RANGE2() macro which
  *  depends on the fact that given range size is a power of 2.