Skip to content

Commit

Permalink
Add support for unmapped and userspace buffers in BUSDMA
Browse files Browse the repository at this point in the history
  • Loading branch information
@strejda
strejda committed Jun 26, 2015
1 parent 447dc4e commit 28e3187
Show file tree
Hide file tree
Showing 4 changed files with 161 additions and 104 deletions.
170 changes: 67 additions & 103 deletions sys/arm/arm/busdma_machdep-v6.c
View file
Expand Up @@ -61,7 +61,7 @@ __FBSDID("$FreeBSD$");

#include <machine/atomic.h>
#include <machine/bus.h>
#include <machine/cpufunc.h>
#include <machine/cpu-v6.h>
#include <machine/md_var.h>

#define MAX_BPAGES 64
Expand Down Expand Up @@ -110,8 +110,8 @@ struct bounce_page {
};

struct sync_list {
vm_offset_t vaddr; /* kva of bounce buffer */
bus_addr_t busaddr; /* Physical address */
vm_offset_t vaddr; /* kva of client data */
bus_addr_t busaddr; /* client physical address */
bus_size_t datacount; /* client data count */
};

Expand Down Expand Up @@ -625,7 +625,8 @@ bus_dma_tag_destroy(bus_dma_tag_t dmat)
return (error);
}

static int allocate_bz_and_pages(bus_dma_tag_t dmat, bus_dmamap_t mapp)
static int
allocate_bz_and_pages(bus_dma_tag_t dmat, bus_dmamap_t mapp)
{
struct bounce_zone *bz;
int maxpages;
Expand Down Expand Up @@ -1043,6 +1044,7 @@ _bus_dmamap_load_phys(bus_dma_tag_t dmat,
{
bus_addr_t curaddr;
bus_size_t sgsize;
struct sync_list *sl;
int error;

if (segs == NULL)
Expand All @@ -1051,7 +1053,7 @@ _bus_dmamap_load_phys(bus_dma_tag_t dmat,
counter_u64_add(maploads_total, 1);
counter_u64_add(maploads_physmem, 1);

if (might_bounce(dmat, map, buflen, buflen)) {
if (might_bounce(dmat, map, (bus_addr_t)buf, buflen)) {
_bus_dmamap_count_phys(dmat, map, buf, buflen, flags);
if (map->pagesneeded != 0) {
counter_u64_add(maploads_bounced, 1);
Expand All @@ -1069,6 +1071,18 @@ _bus_dmamap_load_phys(bus_dma_tag_t dmat,
sgsize = MIN(sgsize, PAGE_SIZE);
curaddr = add_bounce_page(dmat, map, 0, curaddr,
sgsize);
} else {
sl = &map->slist[map->sync_count - 1];
if (map->sync_count == 0 ||
curaddr != sl->busaddr + sl->datacount) {
if (++map->sync_count > dmat->nsegments)
goto cleanup;
sl++;
sl->vaddr = 0;
sl->datacount = sgsize;
sl->busaddr = curaddr;
} else
sl->datacount += sgsize;
}
sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs,
segp);
Expand All @@ -1078,6 +1092,7 @@ _bus_dmamap_load_phys(bus_dma_tag_t dmat,
buflen -= sgsize;
}

cleanup:
/*
* Did we fit?
*/
Expand Down Expand Up @@ -1171,9 +1186,7 @@ _bus_dmamap_load_buffer(bus_dma_tag_t dmat,
} else {
sl = &map->slist[map->sync_count - 1];
if (map->sync_count == 0 ||
#ifdef ARM_L2_PIPT
curaddr != sl->busaddr + sl->datacount ||
#endif
vaddr != sl->vaddr + sl->datacount) {
if (++map->sync_count > dmat->nsegments)
goto cleanup;
Expand Down Expand Up @@ -1251,58 +1264,21 @@ _bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
map->flags &= ~DMAMAP_MBUF;
}

#ifdef notyetbounceuser
/* If busdma uses user pages, then the interrupt handler could
* be use the kernel vm mapping. Both bounce pages and sync list
* do not cross page boundaries.
* Below is a rough sequence that a person would do to fix the
* user page reference in the kernel vmspace. This would be
* done in the dma post routine.
*/
void
_bus_dmamap_fix_user(vm_offset_t buf, bus_size_t len,
pmap_t pmap, int op)
static void
dma_dcache_ops(vm_paddr_t pa, vm_size_t size, int ops)
{
bus_size_t sgsize;
bus_addr_t curaddr;
vm_offset_t va;

/*
* each synclist entry is contained within a single page.
* this would be needed if BUS_DMASYNC_POSTxxxx was implemented
*/
curaddr = pmap_extract(pmap, buf);
va = pmap_dma_map(curaddr);
switch (op) {
case SYNC_USER_INV:
cpu_dcache_wb_range(va, sgsize);
break;

case SYNC_USER_COPYTO:
bcopy((void *)va, (void *)bounce, sgsize);
break;

case SYNC_USER_COPYFROM:
bcopy((void *) bounce, (void *)va, sgsize);
break;

default:
break;
uint32_t len, offset;
vm_page_t m;

offset = pa & PAGE_MASK;
for ( ; size != 0; size -= len, pa += len, offset = 0) {
len = min(PAGE_SIZE - offset, size);
m = PHYS_TO_VM_PAGE(pa);
KASSERT(m != NULL, ("%s: vm_page_t is null for %#x",
__func__, pa));
pmap_dma_dcache_ops(pa, len, m->md.pat_mode, ops);
}

pmap_dma_unmap(va);
}
#endif

#ifdef ARM_L2_PIPT
#define l2cache_wb_range(va, pa, size) cpu_l2cache_wb_range(pa, size)
#define l2cache_wbinv_range(va, pa, size) cpu_l2cache_wbinv_range(pa, size)
#define l2cache_inv_range(va, pa, size) cpu_l2cache_inv_range(pa, size)
#else
#define l2cache_wb_range(va, pa, size) cpu_l2cache_wb_range(va, size)
#define l2cache_wbinv_range(va, pa, size) cpu_l2cache_wbinv_range(va, size)
#define l2cache_inv_range(va, pa, size) cpu_l2cache_inv_range(va, size)
#endif

void
_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
Expand All @@ -1316,9 +1292,6 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
* we're able to test direct userland dma, panic on a map mismatch.
*/
if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) {
if (!pmap_dmap_iscurrent(map->pmap))
panic("_bus_dmamap_sync: wrong user map for bounce sync.");

CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x "
"performing bounce", __func__, dmat, dmat->flags, op);

Expand All @@ -1328,19 +1301,17 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
*/
if (op & BUS_DMASYNC_PREWRITE) {
while (bpage != NULL) {
if (bpage->datavaddr != 0)
if (bpage->datavaddr != 0 &&
pmap_dmap_iscurrent(map->pmap))
bcopy((void *)bpage->datavaddr,
(void *)bpage->vaddr,
bpage->datacount);
else
physcopyout(bpage->dataaddr,
(void *)bpage->vaddr,
bpage->datacount);
cpu_dcache_wb_range((vm_offset_t)bpage->vaddr,
bpage->datacount);
l2cache_wb_range((vm_offset_t)bpage->vaddr,
(vm_offset_t)bpage->busaddr,
bpage->datacount);
dcache_wb_poc(bpage->vaddr,
(vm_paddr_t)bpage->busaddr, bpage->datacount);
bpage = STAILQ_NEXT(bpage, links);
}
dmat->bounce_zone->total_bounced++;
Expand All @@ -1360,11 +1331,8 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
if ((op & BUS_DMASYNC_PREREAD) && !(op & BUS_DMASYNC_PREWRITE)) {
bpage = STAILQ_FIRST(&map->bpages);
while (bpage != NULL) {
cpu_dcache_inv_range((vm_offset_t)bpage->vaddr,
bpage->datacount);
l2cache_inv_range((vm_offset_t)bpage->vaddr,
(vm_offset_t)bpage->busaddr,
bpage->datacount);
dcache_dma_preread(bpage->vaddr,
(vm_paddr_t)bpage->busaddr, bpage->datacount);
bpage = STAILQ_NEXT(bpage, links);
}
}
Expand All @@ -1380,23 +1348,10 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
*/
if (op & BUS_DMASYNC_POSTREAD) {
while (bpage != NULL) {
vm_offset_t startv;
vm_paddr_t startp;
int len;

startv = bpage->vaddr &~ arm_dcache_align_mask;
startp = bpage->busaddr &~ arm_dcache_align_mask;
len = bpage->datacount;

if (startv != bpage->vaddr)
len += bpage->vaddr & arm_dcache_align_mask;
if (len & arm_dcache_align_mask)
len = (len -
(len & arm_dcache_align_mask)) +
arm_dcache_align;
l2cache_inv_range(startv, startp, len);
cpu_dcache_inv_range(startv, len);
if (bpage->datavaddr != 0)
dcache_inv_poc(bpage->vaddr,
(vm_paddr_t)bpage->busaddr, bpage->datacount);
if (bpage->datavaddr != 0 &&
pmap_dmap_iscurrent(map->pmap))
bcopy((void *)bpage->vaddr,
(void *)bpage->datavaddr,
bpage->datacount);
Expand Down Expand Up @@ -1434,9 +1389,6 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
* outer-to-inner for POSTREAD invalidation is not a mistake.
*/
if (map->sync_count != 0) {
if (!pmap_dmap_iscurrent(map->pmap))
panic("_bus_dmamap_sync: wrong user map for sync.");

sl = &map->slist[0];
end = &map->slist[map->sync_count];
CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x "
Expand All @@ -1446,9 +1398,14 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
case BUS_DMASYNC_PREWRITE:
case BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD:
while (sl != end) {
cpu_dcache_wb_range(sl->vaddr, sl->datacount);
l2cache_wb_range(sl->vaddr, sl->busaddr,
sl->datacount);
if (sl->vaddr != 0 &&
pmap_dmap_iscurrent(map->pmap))
dcache_wb_poc(sl->vaddr,
(vm_paddr_t)sl->busaddr,
sl->datacount);
else
dma_dcache_ops((vm_paddr_t)sl->busaddr,
sl->datacount, DDO_WB_POC);
sl++;
}
break;
Expand All @@ -1465,16 +1422,18 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
* they are not aligned to a cacheline.
*/
while (sl != end) {
if (sl->vaddr & arm_dcache_align_mask) {
if (sl->busaddr & cpuinfo.dcache_line_mask) {
KASSERT(map->flags & DMAMAP_MBUF,
("unaligned buffer is not an mbuf"));
cpu_dcache_wb_range(sl->vaddr, 1);
l2cache_wb_range(sl->vaddr,
sl->busaddr, 1);
}
cpu_dcache_inv_range(sl->vaddr, sl->datacount);
l2cache_inv_range(sl->vaddr, sl->busaddr,
sl->datacount);
if (sl->vaddr != 0 &&
pmap_dmap_iscurrent(map->pmap))
dcache_dma_preread_safe(sl->vaddr,
(vm_paddr_t)sl->busaddr,
sl->datacount);
else
dma_dcache_ops((vm_paddr_t)sl->busaddr,
sl->datacount, DDO_PREREAD_SAFE);
sl++;
}
break;
Expand All @@ -1485,9 +1444,14 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
case BUS_DMASYNC_POSTREAD:
case BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE:
while (sl != end) {
l2cache_inv_range(sl->vaddr, sl->busaddr,
sl->datacount);
cpu_dcache_inv_range(sl->vaddr, sl->datacount);
if (sl->vaddr != 0 &&
pmap_dmap_iscurrent(map->pmap))
dcache_inv_poc(sl->vaddr,
(vm_paddr_t)sl->busaddr,
sl->datacount);
else
dma_dcache_ops((vm_paddr_t)sl->busaddr,
sl->datacount, DDO_INV_POC);
sl++;
}
break;
Expand Down
42 changes: 42 additions & 0 deletions sys/arm/arm/pmap-v6-new.c
View file
Expand Up @@ -6076,6 +6076,48 @@ pmap_sync_icache(pmap_t pmap, vm_offset_t va, vm_size_t size)
icache_inv_all();
}

/*
* Do DMA ops on data caches range by physical address.
* The range must be within a single page.
*/
void
pmap_dma_dcache_ops(vm_paddr_t pa, vm_size_t size, vm_memattr_t ma, int ops)
{
struct sysmaps *sysmaps;
vm_offset_t va;

KASSERT(((pa & PAGE_MASK) + size) <= PAGE_SIZE,
("%s: not on single page", __func__));

sched_pin();
sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)];
mtx_lock(&sysmaps->lock);
if (*sysmaps->CMAP3)
panic("%s: CMAP3 busy", __func__);
pte2_store(sysmaps->CMAP3, PTE2_KERN_NG(pa, PTE2_AP_KRW, ma));
tlb_flush_local((vm_offset_t)sysmaps->CADDR3);
va = (vm_offset_t)sysmaps->CADDR3 + (pa & PAGE_MASK);
switch (ops) {
case DDO_WB_POC:
dcache_wb_poc(va, pa, size);
break;
case DDO_INV_POC:
dcache_inv_poc(va, pa, size);
break;
case DDO_PREREAD:
dcache_dma_preread(va, pa, size);
break;
case DDO_PREREAD_SAFE:
dcache_dma_preread_safe(va, pa, size);
break;
default:
panic("%s: unknown ops %d", __func__, ops);
}
pte2_clear(sysmaps->CMAP3);
sched_unpin();
mtx_unlock(&sysmaps->lock);
}

/*
* The implementation of pmap_fault() uses IN_RANGE2() macro which
* depends on the fact that given range size is a power of 2.
Expand Down

0 comments on commit 28e3187

Please sign in to comment.