Skip to content
Browse files

gdev: minor fixes to ioctl wrappers for OS runtime

gdev: added gphysget()
cuda: added cuMemGetPhysAddr()
  • Loading branch information...
1 parent df14f41 commit 2178fc4fa252e3b73636ba781ff61e65ce20e018 Shinpei Kato committed Feb 10, 2012
View
63 common/gdev_api.c
@@ -120,8 +120,8 @@ static int __gmemcpy_to_device_p(gdev_ctx_t *ctx, uint64_t dst_addr, const void
int i;
for (i = 0; i < p_count; i++) {
- dma_addr[i] = gdev_mem_get_addr(bmem[i]);
- dma_buf[i] = gdev_mem_get_buf(bmem[i]);
+ dma_addr[i] = gdev_mem_getaddr(bmem[i]);
+ dma_buf[i] = gdev_mem_getbuf(bmem[i]);
fence[i] = 0;
}
@@ -165,8 +165,8 @@ static int __gmemcpy_to_device_np(gdev_ctx_t *ctx, uint64_t dst_addr, const void
uint32_t dma_size;
int ret = 0;
- dma_addr[0] = gdev_mem_get_addr(bmem[0]);
- dma_buf[0] = gdev_mem_get_buf(bmem[0]);
+ dma_addr[0] = gdev_mem_getaddr(bmem[0]);
+ dma_buf[0] = gdev_mem_getbuf(bmem[0]);
/* copy data by the chunk size. */
offset = 0;
@@ -302,8 +302,8 @@ static int __gmemcpy_from_device_p(gdev_ctx_t *ctx, void *dst_buf, uint64_t src_
int i;
for (i = 0; i < p_count; i++) {
- dma_addr[i] = gdev_mem_get_addr(bmem[i]);
- dma_buf[i] = gdev_mem_get_buf(bmem[i]);
+ dma_addr[i] = gdev_mem_getaddr(bmem[i]);
+ dma_buf[i] = gdev_mem_getbuf(bmem[i]);
fence[i] = 0;
}
@@ -354,8 +354,8 @@ static int __gmemcpy_from_device_np(gdev_ctx_t *ctx, void *dst_buf, uint64_t src
uint32_t dma_size;
int ret = 0;
- dma_addr[0] = gdev_mem_get_addr(bmem[0]);
- dma_buf[0] = gdev_mem_get_buf(bmem[0]);
+ dma_addr[0] = gdev_mem_getaddr(bmem[0]);
+ dma_buf[0] = gdev_mem_getbuf(bmem[0]);
/* copy data by the chunk size. */
offset = 0;
@@ -694,9 +694,9 @@ uint64_t gmalloc(struct gdev_handle *h, uint64_t size)
}
/* size could have been rounded up. */
- gdev->mem_used += gdev_mem_get_size(mem);
+ gdev->mem_used += gdev_mem_getsize(mem);
- return gdev_mem_get_addr(mem);
+ return gdev_mem_getaddr(mem);
fail:
return 0;
@@ -715,7 +715,7 @@ uint64_t gfree(struct gdev_handle *h, uint64_t addr)
if (!(mem = gdev_mem_lookup(vas, addr, GDEV_MEM_DEVICE)))
goto fail;
- size = gdev_mem_get_size(mem);
+ size = gdev_mem_getsize(mem);
gdev_mem_free(mem);
gdev->mem_used -= size;
@@ -742,9 +742,9 @@ void *gmalloc_dma(struct gdev_handle *h, uint64_t size)
goto fail;
/* size could have been rounded up. */
- gdev->dma_mem_used += gdev_mem_get_size(mem);
+ gdev->dma_mem_used += gdev_mem_getsize(mem);
- return gdev_mem_get_buf(mem);
+ return gdev_mem_getbuf(mem);
fail:
return 0;
@@ -763,7 +763,7 @@ uint64_t gfree_dma(struct gdev_handle *h, void *buf)
if (!(mem = gdev_mem_lookup(vas, (uint64_t)buf, GDEV_MEM_DMA)))
goto fail;
- size = gdev_mem_get_size(mem);
+ size = gdev_mem_getsize(mem);
gdev_mem_free(mem);
gdev->dma_mem_used -= size;
@@ -787,7 +787,7 @@ void *gmap(struct gdev_handle *h, uint64_t addr, uint64_t size)
if (!(mem = gdev_mem_lookup(vas, addr, GDEV_MEM_DEVICE)))
goto fail;
- offset = addr - gdev_mem_get_addr(mem);
+ offset = addr - gdev_mem_getaddr(mem);
return gdev_mem_map(mem, offset, size);
fail:
@@ -1057,11 +1057,11 @@ uint64_t gshmat(Ghandle h, int id, uint64_t addr, int flags)
gdev_mutex_lock(&gdev->shm_mutex);
if (!(owner = gdev_shm_lookup(gdev, id)))
goto fail;
- if (!(new = gdev_shm_attach(vas, owner, gdev_mem_get_size(owner))))
+ if (!(new = gdev_shm_attach(vas, owner, gdev_mem_getsize(owner))))
goto fail;
gdev_mutex_unlock(&gdev->shm_mutex);
- return gdev_mem_get_addr(new);
+ return gdev_mem_getaddr(new);
fail:
gdev_mutex_unlock(&gdev->shm_mutex);
@@ -1122,3 +1122,32 @@ int gshmctl(Ghandle h, int id, int cmd, void *buf)
gdev_mutex_unlock(&gdev->shm_mutex);
return ret;
}
+
+/**
+ * gphysget():
+ * get the physical (PCI) bus address associated with buffer pointer @p
+ */
+uint64_t gphysget(Ghandle h, void *p)
+{
+ gdev_vas_t *vas = h->vas;
+ gdev_mem_t *mem;
+ uint32_t type = GDEV_MEM_DMA;
+ uint64_t offset;
+
+ mem = gdev_mem_lookup(vas, (uint64_t)p, type);
+ if (mem)
+ offset = (uint64_t)p - gdev_mem_getaddr(mem);
+ else {
+ type |= GDEV_MEM_DEVICE;
+ mem = gdev_mem_lookup(vas, (uint64_t)p, type);
+ if (mem)
+ offset = (uint64_t)p - (uint64_t)gdev_mem_getbuf(mem);
+ else
+ goto fail;
+ }
+
+ return gdev_mem_phys_getaddr(mem, offset);
+
+fail:
+ return 0;
+}
View
1 common/gdev_api.h
@@ -68,6 +68,7 @@ int gshmget(Ghandle h, int key, uint64_t size, int flags);
uint64_t gshmat(Ghandle h, int id, uint64_t addr, int flags);
int gshmdt(Ghandle h, uint64_t addr);
int gshmctl(Ghandle h, int id, int cmd, void *buf);
+uint64_t gphysget(Ghandle h, void *p);
/**
View
7 common/gdev_arch.h
@@ -80,9 +80,10 @@ void gdev_mem_gc(gdev_vas_t *vas);
void *gdev_mem_map(gdev_mem_t *mem, uint64_t offset, uint64_t size);
void gdev_mem_unmap(gdev_mem_t *mem);
gdev_mem_t *gdev_mem_lookup(gdev_vas_t *vas, uint64_t addr, int type);
-void *gdev_mem_get_buf(gdev_mem_t *mem);
-uint64_t gdev_mem_get_addr(gdev_mem_t *mem);
-uint64_t gdev_mem_get_size(gdev_mem_t *mem);
+void *gdev_mem_getbuf(gdev_mem_t *mem);
+uint64_t gdev_mem_getaddr(gdev_mem_t *mem);
+uint64_t gdev_mem_getsize(gdev_mem_t *mem);
+uint64_t gdev_mem_phys_getaddr(gdev_mem_t *mem, uint64_t offset);
int gdev_shm_create(struct gdev_device *gdev, gdev_vas_t *vas, int key, uint64_t size, int flags);
int gdev_shm_destroy_mark(struct gdev_device *gdev, gdev_mem_t *owner);
gdev_mem_t *gdev_shm_attach(gdev_vas_t *vas, gdev_mem_t *mem, uint64_t size);
View
6 common/gdev_ioctl_def.h
@@ -52,6 +52,7 @@
#define GDEV_IOCTL_GSHMAT 0x117
#define GDEV_IOCTL_GSHMDT 0x118
#define GDEV_IOCTL_GSHMCTL 0x119
+#define GDEV_IOCTL_GPHYSGET 0x120
struct gdev_ioctl_mem {
uint64_t addr;
@@ -103,4 +104,9 @@ struct gdev_ioctl_map {
uint64_t size;
};
+struct gdev_ioctl_phys {
+ uint64_t addr;
+ uint64_t phys;
+};
+
#endif
View
3 common/gdev_nvidia.h
@@ -183,8 +183,8 @@ struct gdev_mem {
struct gdev_mem *swap_mem; /* device memory for temporal swap */
void *swap_buf; /* host buffer for swap */
int evicted; /* 1 if evicted, 0 otherwise */
- uint64_t addr; /* virtual memory address */
uint64_t size; /* memory size */
+ uint64_t addr; /* virtual memory address */
int type; /* device or host dma? */
void *map; /* memory-mapped buffer */
int map_users; /* # of users referencing the map */
@@ -235,6 +235,7 @@ struct gdev_mem *gdev_raw_mem_share(struct gdev_vas *vas, struct gdev_mem *mem,
void gdev_raw_mem_unshare(struct gdev_mem *mem);
void *gdev_raw_mem_map(struct gdev_mem *mem);
void gdev_raw_mem_unmap(struct gdev_mem *mem, void *map);
+uint64_t gdev_raw_mem_phys_getaddr(struct gdev_mem *mem, uint64_t offset);
uint32_t gdev_raw_read32(struct gdev_mem *mem, uint64_t addr);
void gdev_raw_write32(struct gdev_mem *mem, uint64_t addr, uint32_t val);
int gdev_raw_read(struct gdev_mem *mem, void *buf, uint64_t addr, uint32_t size);
View
14 common/gdev_nvidia_mem.c
@@ -32,8 +32,8 @@
void gdev_nvidia_mem_init(struct gdev_mem *mem, struct gdev_vas *vas, uint64_t addr, uint64_t size, void *map, int type)
{
mem->vas = vas;
- mem->addr = addr;
mem->size = size;
+ mem->addr = addr;
mem->map = map;
mem->type = type;
mem->evicted = 0;
@@ -291,20 +291,26 @@ struct gdev_mem *gdev_mem_lookup(struct gdev_vas *vas, uint64_t addr, int type)
}
/* get host DMA buffer (could be memory-mapped buffer for device memory). */
-void *gdev_mem_get_buf(struct gdev_mem *mem)
+void *gdev_mem_getbuf(struct gdev_mem *mem)
{
return mem->map;
}
/* get virtual memory address. */
-uint64_t gdev_mem_get_addr(struct gdev_mem *mem)
+uint64_t gdev_mem_getaddr(struct gdev_mem *mem)
{
return mem->addr;
}
/* get allocated memory size. */
-uint64_t gdev_mem_get_size(struct gdev_mem *mem)
+uint64_t gdev_mem_getsize(struct gdev_mem *mem)
{
return mem->size;
}
+/* get physical bus address. */
+uint64_t gdev_mem_phys_getaddr(struct gdev_mem *mem, uint64_t offset)
+{
+ return gdev_raw_mem_phys_getaddr(mem, offset);
+}
+
View
2 cuda/driver_api/cuda.h
@@ -765,6 +765,8 @@ CUresult cuMemcpyDtoD(CUdeviceptr dstDevice, CUdeviceptr srcDevice, unsigned int
/* Memory mapping - Gdev extension */
CUresult cuMemMap(void **buf, CUdeviceptr dptr, unsigned int bytesize);
CUresult cuMemUnmap(void *buf);
+/* Memory mapped address - Gdev extension */
+CUresult cuMemGetPhysAddr(unsigned long long *addr, void *p);
/* Inter-Process Communication (IPC) - Gdev extension */
CUresult cuShmGet(int *ptr, int key, size_t size, int flags);
View
31 cuda/driver_api/memory.c
@@ -415,3 +415,34 @@ CUresult cuMemUnmap(void *buf)
return CUDA_SUCCESS;
}
+
+/**
+ * Gdev extension: returns physical bus address associated to user buffer.
+ * Note that the address is contiguous only within the page boundary.
+ *
+ * Parameters:
+ * addr - Physical bus address obtained
+ * p - Pointer to user buffer
+ *
+ * Returns:
+ * CUDA_SUCCESS, CUDA_ERROR_DEINITIALIZED, CUDA_ERROR_NOT_INITIALIZED,
+ * CUDA_ERROR_INVALID_CONTEXT, CUDA_ERROR_INVALID_VALUE
+ */
+CUresult cuMemGetPhysAddr(unsigned long long *addr, void *p)
+{
+ Ghandle handle;
+
+ if (!gdev_initialized)
+ return CUDA_ERROR_NOT_INITIALIZED;
+ if (!gdev_ctx_current)
+ return CUDA_ERROR_INVALID_CONTEXT;
+ if (!addr || !p)
+ return CUDA_ERROR_INVALID_VALUE;
+
+ handle = gdev_ctx_current->gdev_handle;
+
+ if (!(*addr = gphysget(handle, p)))
+ return CUDA_ERROR_UNKNOWN;
+
+ return CUDA_SUCCESS;
+}
View
2 driver/gdev/gdev_fops.c
@@ -124,6 +124,8 @@ static int gdev_ioctl
return gdev_ioctl_gshmdt(handle, arg);
case GDEV_IOCTL_GSHMCTL:
return gdev_ioctl_gshmctl(handle, arg);
+ case GDEV_IOCTL_GPHYSGET:
+ return gdev_ioctl_gphysget(handle, arg);
default:
GDEV_PRINT("Ioctl command 0x%x is not supported.\n", cmd);
return -EINVAL;
View
15 driver/gdev/gdev_ioctl.c
@@ -421,3 +421,18 @@ int gdev_ioctl_gshmctl(Ghandle handle, unsigned long arg)
return gshmctl(handle, s.id, s.cmd, (void *)&ds);
}
+int gdev_ioctl_gphysget(Ghandle handle, unsigned long arg)
+{
+ struct gdev_ioctl_phys p;
+
+ if (copy_from_user(&p, (void __user *)arg, sizeof(p)))
+ return -EFAULT;
+
+ if (!(p.phys = gphysget(handle, (void *)p.addr)))
+ return -EINVAL;
+
+ if (copy_to_user((void __user *)arg, &p, sizeof(p)))
+ return -EFAULT;
+
+ return 0;
+}
View
5 driver/gdev/gdev_ioctl.h
@@ -36,9 +36,9 @@ int gdev_ioctl_gmalloc(Ghandle h, unsigned long arg);
int gdev_ioctl_gfree(Ghandle h, unsigned long arg);
int gdev_ioctl_gmalloc_dma(Ghandle h, unsigned long arg);
int gdev_ioctl_gfree_dma(Ghandle h, unsigned long arg);
-int gdev_ioctl_gmap(Ghandle handle, unsigned long arg);
+int gdev_ioctl_gmap(Ghandle h, unsigned long arg);
+int gdev_ioctl_gunmap(Ghandle h, unsigned long arg);
int gdev_ioctl_gmemcpy_to_device(Ghandle h, unsigned long arg);
-int gdev_ioctl_gunmap(Ghandle handle, unsigned long arg);
int gdev_ioctl_gmemcpy_to_device_async(Ghandle h, unsigned long arg);
int gdev_ioctl_gmemcpy_from_device(Ghandle h, unsigned long arg);
int gdev_ioctl_gmemcpy_from_device_async(Ghandle h, unsigned long arg);
@@ -52,5 +52,6 @@ int gdev_ioctl_gshmget(Ghandle h, unsigned long arg);
int gdev_ioctl_gshmat(Ghandle h, unsigned long arg);
int gdev_ioctl_gshmdt(Ghandle h, unsigned long arg);
int gdev_ioctl_gshmctl(Ghandle h, unsigned long arg);
+int gdev_ioctl_gphysget(Ghandle h, unsigned long arg);
#endif
View
8 driver/pscnv/nv50_vm.c
@@ -231,6 +231,13 @@ nv84_vm_bar_flush(struct drm_device *dev) {
}
}
+uint64_t nv50_vm_phys_getaddr(struct pscnv_vspace *vs, struct pscnv_bo *bo, uint64_t addr)
+{
+ struct drm_device *dev = vs->dev;
+ NV_ERROR(dev, "nv50_vm_phys_getaddr(): Not supported yet!\n");
+ return 0;
+}
+
int nv50_vm_read32(struct pscnv_vspace *vs, struct pscnv_bo *bo, uint64_t addr, uint32_t *ptr)
{
struct drm_device *dev = vs->dev;
@@ -280,6 +287,7 @@ nv50_vm_init(struct drm_device *dev) {
vme->base.bar_flush = nv50_vm_bar_flush;
else
vme->base.bar_flush = nv84_vm_bar_flush;
+ vme->base.phys_getaddr = nv50_vm_phys_getaddr;
vme->base.read32 = nv50_vm_read32;
vme->base.write32 = nv50_vm_write32;
vme->base.read = nv50_vm_read;
View
104 driver/pscnv/nvc0_vm.c
@@ -352,28 +352,34 @@ int nvc0_vm_map_kernel(struct pscnv_bo *bo) {
return pscnv_vspace_map(vme->bar3vm, bo, 0, dev_priv->ramin_size, 0, &bo->map3);
}
-int nvc0_vm_read32(struct pscnv_vspace *vs, struct pscnv_bo *bo, uint64_t addr, uint32_t *ptr)
+uint64_t nvc0_vm_phys_getaddr(struct pscnv_vspace *vs, struct pscnv_bo *bo, uint64_t addr)
{
- struct drm_device *dev = vs->dev;
- struct drm_nouveau_private *dev_priv = dev->dev_private;
int s = (bo->flags & PSCNV_GEM_MEMTYPE_MASK) != PSCNV_GEM_VRAM_LARGE;
uint32_t psh = s ? NVC0_SPAGE_SHIFT : NVC0_LPAGE_SHIFT;
unsigned int pde = NVC0_PDE(addr);
unsigned int pte = (addr & NVC0_VM_BLOCK_MASK) >> psh;
uint64_t vm_block = pte << psh;
uint64_t offset = (addr & NVC0_VM_BLOCK_MASK) - vm_block;
struct nvc0_pgt *pt = nvc0_vspace_pgt(vs, pde);
+
+ return ((nv_rv32(pt->bo[s], pte * 8) >> 4) << 12) + offset;
+}
+
+int nvc0_vm_read32(struct pscnv_vspace *vs, struct pscnv_bo *bo, uint64_t addr, uint32_t *ptr)
+{
+ struct drm_device *dev = vs->dev;
+ struct drm_nouveau_private *dev_priv = dev->dev_private;
+ uint32_t pmem = 0x700000;
uint64_t phys;
uint32_t val;
- phys = ((nv_rv32(pt->bo[s], pte * 8) >> 4) << 12) + offset;
-
spin_lock(&dev_priv->pramin_lock);
+ phys = nvc0_vm_phys_getaddr(vs, bo, addr);
if (phys >> 16 != dev_priv->pramin_start) {
dev_priv->pramin_start = phys >> 16;
nv_wr32(dev, 0x1700, phys >> 16);
}
- val = nv_rd32(dev, 0x700000 + (phys & 0xffff));
+ val = nv_rd32(dev, pmem + (phys & 0xffff));
spin_unlock(&dev_priv->pramin_lock);
*ptr = val;
@@ -385,23 +391,16 @@ int nvc0_vm_write32(struct pscnv_vspace *vs, struct pscnv_bo *bo, uint64_t addr,
{
struct drm_device *dev = vs->dev;
struct drm_nouveau_private *dev_priv = dev->dev_private;
- int s = (bo->flags & PSCNV_GEM_MEMTYPE_MASK) != PSCNV_GEM_VRAM_LARGE;
- uint32_t psh = s ? NVC0_SPAGE_SHIFT : NVC0_LPAGE_SHIFT;
- unsigned int pde = NVC0_PDE(addr);
- unsigned int pte = (addr & NVC0_VM_BLOCK_MASK) >> psh;
- uint64_t vm_block = pte << psh;
- uint64_t offset = (addr & NVC0_VM_BLOCK_MASK) - vm_block;
- struct nvc0_pgt *pt = nvc0_vspace_pgt(vs, pde);
+ uint32_t pmem = 0x700000;
uint64_t phys;
- phys = ((nv_rv32(pt->bo[s], pte * 8) >> 4) << 12) + offset;
-
spin_lock(&dev_priv->pramin_lock);
+ phys = nvc0_vm_phys_getaddr(vs, bo, addr);
if ((phys >> 16) != dev_priv->pramin_start) {
dev_priv->pramin_start = phys >> 16;
nv_wr32(dev, 0x1700, phys >> 16);
}
- nv_wr32(dev, 0x700000 + (phys & 0xffff), val);
+ nv_wr32(dev, pmem + (phys & 0xffff), val);
spin_unlock(&dev_priv->pramin_lock);
return 0;
@@ -411,40 +410,29 @@ int nvc0_vm_read(struct pscnv_vspace *vs, struct pscnv_bo *bo, uint64_t addr, vo
{
struct drm_device *dev = vs->dev;
struct drm_nouveau_private *dev_priv = dev->dev_private;
- int s = (bo->flags & PSCNV_GEM_MEMTYPE_MASK) != PSCNV_GEM_VRAM_LARGE;
- uint32_t psh = s ? NVC0_SPAGE_SHIFT : NVC0_LPAGE_SHIFT;
- unsigned int pde = NVC0_PDE(addr);
- unsigned int pte = (addr & NVC0_VM_BLOCK_MASK) >> psh;
- uint64_t vm_block = pte << psh;
- uint64_t offset = (addr & NVC0_VM_BLOCK_MASK) - vm_block;
- struct nvc0_pgt *pt = nvc0_vspace_pgt(vs, pde);
- uint64_t phys;
+ uint32_t pmem = 0x700000;
uint32_t wsize;
- int i;
-
- phys = ((nv_rv32(pt->bo[s], pte * 8) >> 4) << 12) + offset;
+ uint64_t phys;
- spin_lock(&dev_priv->pramin_lock);
do {
- wsize = 0x10000;
+ spin_lock(&dev_priv->pramin_lock);
+ phys = nvc0_vm_phys_getaddr(vs, bo, addr);
+ wsize = PAGE_SIZE;
if (wsize > size)
wsize = size;
dev_priv->pramin_start = phys >> 16;
nv_wr32(dev, 0x1700, phys >> 16);
- if (wsize > 0x1000) {
- memcpy_fromio(buf, dev_priv->mmio + 0x700000 + (phys & 0xffff), size);
- }
- else {
- for (i = 0; i < wsize / 4; i++) {
- ((uint32_t*)buf)[i] =
- nv_rd32(dev, 0x700000 + (phys & 0xffff) + i * 4);
- }
- }
+ memcpy_fromio(buf, dev_priv->mmio + pmem + (phys & 0xffff), size);
+ /*
+ *int i;
+ *for (i = 0; i < wsize / 4; i++)
+ *((uint32_t*)buf)[i] = nv_rd32(dev, pmem + (phys & 0xffff) + i * 4);
+ */
size -= wsize;
- phys += wsize;
+ addr += wsize;
buf += wsize;
+ spin_unlock(&dev_priv->pramin_lock);
} while (size);
- spin_unlock(&dev_priv->pramin_lock);
return 0;
}
@@ -453,40 +441,29 @@ int nvc0_vm_write(struct pscnv_vspace *vs, struct pscnv_bo *bo, uint64_t addr, c
{
struct drm_device *dev = vs->dev;
struct drm_nouveau_private *dev_priv = dev->dev_private;
- int s = (bo->flags & PSCNV_GEM_MEMTYPE_MASK) != PSCNV_GEM_VRAM_LARGE;
- uint32_t psh = s ? NVC0_SPAGE_SHIFT : NVC0_LPAGE_SHIFT;
- unsigned int pde = NVC0_PDE(addr);
- unsigned int pte = (addr & NVC0_VM_BLOCK_MASK) >> psh;
- uint64_t vm_block = pte << psh;
- uint64_t offset = (addr & NVC0_VM_BLOCK_MASK) - vm_block;
- struct nvc0_pgt *pt = nvc0_vspace_pgt(vs, pde);
+ uint32_t pmem = 0x700000;
uint64_t phys;
uint32_t wsize;
- int i;
- phys = ((nv_rv32(pt->bo[s], pte * 8) >> 4) << 12) + offset;
-
- spin_lock(&dev_priv->pramin_lock);
do {
- wsize = 0x10000;
+ spin_lock(&dev_priv->pramin_lock);
+ phys = nvc0_vm_phys_getaddr(vs, bo, addr);
+ wsize = PAGE_SIZE;
if (wsize > size)
wsize = size;
dev_priv->pramin_start = phys >> 16;
nv_wr32(dev, 0x1700, phys >> 16);
- if (wsize > 0x1000) {
- memcpy_toio(dev_priv->mmio + 0x700000 + (phys & 0xffff), buf, wsize);
- }
- else {
- for (i = 0; i < wsize / 4; i++) {
- nv_wr32(dev, 0x700000 + (phys & 0xffff) + i * 4,
- ((uint32_t*)buf)[i]);
- }
- }
+ memcpy_toio(dev_priv->mmio + pmem + (phys & 0xffff), buf, wsize);
+ /*
+ *int i;
+ *for (i = 0; i < wsize / 4; i++)
+ * nv_wr32(dev, pmem + (phys & 0xffff) + i * 4, ((uint32_t*)buf)[i]);
+ */
size -= wsize;
- phys += wsize;
+ addr += wsize;
buf += wsize;
+ spin_unlock(&dev_priv->pramin_lock);
} while (size);
- spin_unlock(&dev_priv->pramin_lock);
return 0;
}
@@ -509,6 +486,7 @@ nvc0_vm_init(struct drm_device *dev) {
vme->base.map_user = nvc0_vm_map_user;
vme->base.map_kernel = nvc0_vm_map_kernel;
vme->base.bar_flush = nv84_vm_bar_flush;
+ vme->base.phys_getaddr = nvc0_vm_phys_getaddr;
vme->base.read32 = nvc0_vm_read32;
vme->base.write32 = nvc0_vm_write32;
vme->base.read = nvc0_vm_read;
View
10 driver/pscnv/pscnv_drm.h
@@ -164,6 +164,14 @@ struct drm_pscnv_vm_map {
uint64_t map_handle; /* > < */
};
+struct drm_pscnv_phys_getaddr {
+ uint32_t vid; /* < */
+ uint32_t handle; /* < */
+ uint64_t addr; /* < */
+ uint32_t offset; /* < */
+ uint64_t phys; /* > */
+};
+
#define DRM_PSCNV_GETPARAM 0x00 /* get some information from the card */
#define DRM_PSCNV_GEM_NEW 0x20 /* create a new BO */
#define DRM_PSCNV_GEM_INFO 0x21 /* get info about a BO */
@@ -184,6 +192,7 @@ struct drm_pscnv_vm_map {
#define DRM_PSCNV_VM_WRITE 0x2f /* Write to virtual memory */
#define DRM_PSCNV_VM_MAP 0x30 /* Map virtual memory */
#define DRM_PSCNV_VM_UNMAP 0x31 /* Unmap virtual memory */
+#define DRM_PSCNV_PHYS_GETADDR 0x32 /* Get physical address */
#define DRM_IOCTL_PSCNV_GETPARAM DRM_IOWR(DRM_COMMAND_BASE + DRM_PSCNV_GETPARAM, struct drm_pscnv_getparam)
#define DRM_IOCTL_PSCNV_GEM_NEW DRM_IOWR(DRM_COMMAND_BASE + DRM_PSCNV_GEM_NEW, struct drm_pscnv_gem_info)
@@ -205,5 +214,6 @@ struct drm_pscnv_vm_map {
#define DRM_IOCTL_PSCNV_VM_WRITE DRM_IOW(DRM_COMMAND_BASE + DRM_PSCNV_VM_WRITE, struct drm_pscnv_vm_rw)
#define DRM_IOCTL_PSCNV_VM_MAP DRM_IOW(DRM_COMMAND_BASE + DRM_PSCNV_VM_MAP, struct drm_pscnv_vm_map)
#define DRM_IOCTL_PSCNV_VM_UNMAP DRM_IOW(DRM_COMMAND_BASE + DRM_PSCNV_VM_UNMAP, struct drm_pscnv_vm_map)
+#define DRM_IOCTL_PSCNV_PHYS_GETADDR DRM_IOW(DRM_COMMAND_BASE + DRM_PSCNV_PHYS_GETADDR, struct drm_pscnv_phys_getaddr)
#endif /* __PSCNV_DRM_H__ */
View
111 driver/pscnv/pscnv_gdev.c
@@ -485,6 +485,69 @@ void gdev_raw_mem_unshare(struct gdev_mem *mem)
kfree(mem);
}
+/* map device memory to host DMA memory. */
+void *gdev_raw_mem_map(struct gdev_mem *mem)
+{
+ struct gdev_vas *vas = mem->vas;
+ struct gdev_device *gdev = vas->gdev;
+ struct drm_device *drm = (struct drm_device *) gdev->priv;
+ struct drm_nouveau_private *dev_priv = drm->dev_private;
+ struct pscnv_bo *bo = mem->bo;
+ unsigned long bar1_start = pci_resource_start(drm->pdev, 1);
+ void *map;
+
+ if (dev_priv->vm->map_user(bo))
+ goto fail_map_user;
+ if (!(map = ioremap(bar1_start + bo->map1->start, bo->size)))
+ goto fail_ioremap;
+
+ bo->flags |= PSCNV_GEM_MAPPABLE;
+
+ return map;
+
+fail_ioremap:
+ GDEV_PRINT("Failed to map PCI BAR1\n");
+ pscnv_vspace_unmap_node(bo->map1);
+fail_map_user:
+ GDEV_PRINT("Failed to map host and device memory\n");
+
+ return NULL;
+}
+
+/* unmap device memory from host DMA memory. */
+void gdev_raw_mem_unmap(struct gdev_mem *mem, void *map)
+{
+ struct pscnv_bo *bo = mem->bo;
+
+ iounmap(map);
+ pscnv_vspace_unmap_node(bo->map1);
+ bo->flags &= ~PSCNV_GEM_MAPPABLE;
+}
+
+/* get physical bus address. */
+uint64_t gdev_raw_mem_phys_getaddr(struct gdev_mem *mem, uint64_t offset)
+{
+ struct gdev_vas *vas = mem->vas;
+ struct gdev_device *gdev = vas->gdev;
+ struct drm_device *drm = (struct drm_device *) gdev->priv;
+ struct drm_nouveau_private *dev_priv = drm->dev_private;
+ struct pscnv_vspace *vspace = vas->pvas;
+ struct pscnv_bo *bo = mem->bo;
+ int page = offset / PAGE_SIZE;
+ uint32_t x = offset - page * PAGE_SIZE;
+ uint32_t flags = bo->flags;
+
+ if (flags & PSCNV_GEM_MAPPABLE) {
+ if (flags & PSCNV_GEM_SYSRAM_SNOOP)
+ return bo->dmapages[page] + x;
+ else
+ return pci_resource_start(drm->pdev, 1) + bo->map1->start + x;
+ }
+ else {
+ return dev_priv->vm->phys_getaddr(vspace, bo, mem->addr + offset);
+ }
+}
+
uint32_t gdev_raw_read32(struct gdev_mem *mem, uint64_t addr)
{
struct gdev_vas *vas = mem->vas;
@@ -537,19 +600,17 @@ int gdev_raw_read(struct gdev_mem *mem, void *buf, uint64_t addr, uint32_t size)
struct drm_device *drm = (struct drm_device *) gdev->priv;
struct drm_nouveau_private *dev_priv = drm->dev_private;
uint64_t offset = addr - mem->addr;
- int ret;
if (mem->map) {
memcpy_fromio(buf, mem->map + offset, size);
- return 0;
}
else {
mutex_lock(&vspace->lock);
- ret = dev_priv->vm->read(vspace, bo, addr, buf, size);
+ dev_priv->vm->read(vspace, bo, addr, buf, size);
mutex_unlock(&vspace->lock);
}
- return ret;
+ return 0;
}
int gdev_raw_write(struct gdev_mem *mem, uint64_t addr, const void *buf, uint32_t size)
@@ -561,53 +622,15 @@ int gdev_raw_write(struct gdev_mem *mem, uint64_t addr, const void *buf, uint32_
struct drm_device *drm = (struct drm_device *) gdev->priv;
struct drm_nouveau_private *dev_priv = drm->dev_private;
uint64_t offset = addr - mem->addr;
- int ret;
if (mem->map) {
memcpy_toio(mem->map + offset, buf, size);
- return 0;
}
else {
mutex_lock(&vspace->lock);
- ret = dev_priv->vm->write(vspace, bo, addr, buf, size);
+ dev_priv->vm->write(vspace, bo, addr, buf, size);
mutex_unlock(&vspace->lock);
}
- return ret;
-}
-
-/* map device memory to host DMA memory. */
-void *gdev_raw_mem_map(struct gdev_mem *mem)
-{
- struct gdev_vas *vas = mem->vas;
- struct gdev_device *gdev = vas->gdev;
- struct drm_device *drm = (struct drm_device *) gdev->priv;
- struct drm_nouveau_private *dev_priv = drm->dev_private;
- struct pscnv_bo *bo = mem->bo;
- unsigned long bar1_start = pci_resource_start(drm->pdev, 1);
- void *map;
-
- if (dev_priv->vm->map_user(bo))
- goto fail_map_user;
- if (!(map = ioremap(bar1_start + bo->map1->start, bo->size)))
- goto fail_ioremap;
-
- return map;
-
-fail_ioremap:
- GDEV_PRINT("Failed to map PCI BAR1\n");
- pscnv_vspace_unmap_node(bo->map1);
-fail_map_user:
- GDEV_PRINT("Failed to map host and device memory\n");
-
- return NULL;
-}
-
-/* unmap device memory from host DMA memory. */
-void gdev_raw_mem_unmap(struct gdev_mem *mem, void *map)
-{
- struct pscnv_bo *bo = mem->bo;
-
- iounmap(map);
- pscnv_vspace_unmap_node(bo->map1);
+ return 0;
}
View
45 driver/pscnv/pscnv_ioctl.c
@@ -656,6 +656,7 @@ int pscnv_ioctl_vm_map(struct drm_device *dev, void *data, struct drm_file *file
/* map the buffer object to BAR1. */
ret = dev_priv->vm->map_user(bo);
+ bo->flags |= PSCNV_GEM_MAPPABLE;
pscnv_vspace_unref(vs);
@@ -689,12 +690,54 @@ int pscnv_ioctl_vm_unmap(struct drm_device *dev, void *data, struct drm_file *fi
if (dev_priv->vm_ok && bo->map1)
pscnv_vspace_unmap_node(bo->map1);
+ bo->flags &= ~PSCNV_GEM_MAPPABLE;
pscnv_vspace_unref(vs);
return 0;
}
+int pscnv_ioctl_phys_getaddr(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+ struct drm_pscnv_phys_getaddr *req = data;
+ struct drm_nouveau_private *dev_priv = dev->dev_private;
+ struct drm_gem_object *obj;
+ struct pscnv_bo *bo;
+ struct pscnv_vspace *vs;
+ int page;
+ uint32_t x;
+
+ NOUVEAU_CHECK_INITIALISED_WITH_RETURN;
+
+ vs = pscnv_get_vspace(dev, file_priv, req->vid);
+ if (!vs)
+ return -ENOENT;
+
+ obj = drm_gem_object_lookup(dev, file_priv, req->handle);
+ if (!obj) {
+ pscnv_vspace_unref(vs);
+ return -EBADF;
+ }
+
+ bo = obj->driver_private;
+
+ page = req->offset / PAGE_SIZE;
+ x = req->offset - page * PAGE_SIZE;
+
+ if (bo->flags & PSCNV_GEM_MAPPABLE) {
+ if (bo->flags & PSCNV_GEM_SYSRAM_SNOOP)
+ req->phys = bo->dmapages[page] + x;
+ else
+ req->phys = pci_resource_start(dev->pdev, 1) + bo->map1->start + x;
+ }
+ else {
+ req->phys = dev_priv->vm->phys_getaddr(vs, bo, req->addr + req->offset);
+ }
+
+ pscnv_vspace_unref(vs);
+
+ return 0;
+}
#ifdef PSCNV_KAPI_DRM_IOCTL_DEF_DRV
struct drm_ioctl_desc nouveau_ioctls[] = {
@@ -717,6 +760,7 @@ struct drm_ioctl_desc nouveau_ioctls[] = {
DRM_IOCTL_DEF_DRV(PSCNV_VM_WRITE, pscnv_ioctl_vm_write, DRM_UNLOCKED),
DRM_IOCTL_DEF_DRV(PSCNV_VM_MAP, pscnv_ioctl_vm_map, DRM_UNLOCKED),
DRM_IOCTL_DEF_DRV(PSCNV_VM_UNMAP, pscnv_ioctl_vm_unmap, DRM_UNLOCKED),
+ DRM_IOCTL_DEF_DRV(PSCNV_PHYS_GETADDR, pscnv_ioctl_phys_getaddr, DRM_UNLOCKED),
};
#elif defined(PSCNV_KAPI_DRM_IOCTL_DEF)
struct drm_ioctl_desc nouveau_ioctls[] = {
@@ -739,6 +783,7 @@ struct drm_ioctl_desc nouveau_ioctls[] = {
DRM_IOCTL_DEF(DRM_PSCNV_VM_WRITE, pscnv_ioctl_vm_write, DRM_UNLOCKED),
DRM_IOCTL_DEF(DRM_PSCNV_VM_MAP, pscnv_ioctl_vm_map, DRM_UNLOCKED),
DRM_IOCTL_DEF(DRM_PSCNV_VM_UNMAP, pscnv_ioctl_vm_unmap, DRM_UNLOCKED),
+ DRM_IOCTL_DEF(DRM_PSCNV_PHYS_GETADDR, pscnv_ioctl_phys_getaddr, DRM_UNLOCKED),
};
#else
#error "Unknown IOCTLDEF method."
View
1 driver/pscnv/pscnv_vm.h
@@ -55,6 +55,7 @@ struct pscnv_vm_engine {
int (*map_user) (struct pscnv_bo *);
int (*map_kernel) (struct pscnv_bo *);
void (*bar_flush) (struct drm_device *dev);
+ uint64_t (*phys_getaddr) (struct pscnv_vspace *vs, struct pscnv_bo *bo, uint64_t addr);
int (*read32) (struct pscnv_vspace *vs, struct pscnv_bo *bo, uint64_t addr, uint32_t *ptr);
int (*write32) (struct pscnv_vspace *vs, struct pscnv_bo *bo, uint64_t addr, uint32_t val);
int (*read) (struct pscnv_vspace *vs, struct pscnv_bo *bo, uint64_t addr, void *buf, uint32_t size);
View
51 runtime/kernel/gdev_lib.c
@@ -119,7 +119,7 @@ void *gmalloc_dma(struct gdev_handle *h, uint64_t size)
goto fail_malloc;
gdev_list_init(&bo->list_entry, bo);
gdev_list_add(&bo->list_entry, &h->map_bo_list);
- bo->addr = mem.addr;
+ bo->addr = mem.addr; /* buffer pointer address valid in OS-space */
bo->size = size; /* could be different from mem.size */
bo->buf = buf;
@@ -140,7 +140,7 @@ uint64_t gfree_dma(struct gdev_handle *h, void *buf)
int fd = h->fd;
gdev_list_for_each (bo, &h->map_bo_list, list_entry) {
- if (bo && (bo->buf == buf)) {
+ if (bo->buf == buf) {
goto free;
}
}
@@ -179,7 +179,7 @@ void *gmap(struct gdev_handle *h, uint64_t addr, uint64_t size)
goto fail_malloc;
gdev_list_init(&bo->list_entry, bo);
gdev_list_add(&bo->list_entry, &h->map_bo_list);
- bo->addr = map.buf; /* save buffer address but not device address */
+ bo->addr = map.buf; /* buffer pointer address valid in OS-space */
bo->size = size;
bo->buf = buf;
@@ -200,7 +200,7 @@ int gunmap(struct gdev_handle *h, void *buf)
int fd = h->fd;
gdev_list_for_each (bo, &h->map_bo_list, list_entry) {
- if (bo && (bo->buf == buf)) {
+ if (bo->buf == buf) {
goto unmap;
}
}
@@ -221,17 +221,23 @@ static int __gmemcpy_to_device(struct gdev_handle *h, uint64_t dst_addr, const v
{
struct gdev_map_bo *bo;
struct gdev_ioctl_dma dma;
+ uint64_t src_addr = (uint64_t)src_buf;
+ uint64_t buf_addr;
int fd = h->fd;
+ /* look up if @src_buf is allocated on host DMA buffer already. */
gdev_list_for_each (bo, &h->map_bo_list, list_entry) {
- if (bo && (bo->buf == src_buf))
+ buf_addr = (uint64_t)bo->buf;
+ if ((src_addr >= buf_addr) && (src_addr < buf_addr + bo->size))
break;
}
dma.dst_addr = dst_addr;
if (bo)
- dma.src_buf = (void *) bo->addr;
+ /* this is "PCI-space" host address */
+ dma.src_buf = (void *)(bo->addr + (src_addr - buf_addr));
else
+ /* this is "user-space" buffer */
dma.src_buf = src_buf;
dma.size = size;
dma.id = id;
@@ -253,17 +259,23 @@ static int __gmemcpy_from_device(struct gdev_handle *h, void *dst_buf, uint64_t
{
struct gdev_map_bo *bo;
struct gdev_ioctl_dma dma;
+ uint64_t dst_addr = (uint64_t)dst_buf;
+ uint64_t buf_addr;
int fd = h->fd;
+ /* look up if @dst_buf is allocated on host DMA buffer already. */
gdev_list_for_each (bo, &h->map_bo_list, list_entry) {
- if (bo && (bo->buf == dst_buf))
+ buf_addr = (uint64_t)bo->buf;
+ if ((dst_addr >= buf_addr) && (dst_addr < buf_addr + bo->size))
break;
}
dma.src_addr = src_addr;
if (bo)
- dma.dst_buf = (void *) bo->addr;
+ /* this is "PCI-space" host address */
+ dma.dst_buf = (void *)(bo->addr + (dst_addr - buf_addr));
else
+ /* this is "user-space" buffer */
dma.dst_buf = dst_buf;
dma.size = size;
dma.id = id;
@@ -406,3 +418,26 @@ int gshmctl(struct gdev_handle *h, int id, int cmd, void *buf)
return 0;
}
+
+uint64_t gphysget(struct gdev_handle *h, void *p)
+{
+ struct gdev_map_bo *bo;
+ struct gdev_ioctl_phys phys;
+ int fd = h->fd;
+ uint64_t p_addr = (uint64_t)p;
+ uint64_t buf_addr;
+
+ gdev_list_for_each (bo, &h->map_bo_list, list_entry) {
+ buf_addr = (uint64_t)bo->buf;
+ if ((p_addr >= buf_addr) && (p_addr < buf_addr + bo->size))
+ goto physget;
+ }
+ return 0;
+
+physget:
+ /* bo->addr is buffer pointer address valid in OS-space. */
+ phys.addr = bo->addr + (p_addr - buf_addr);
+ ioctl(fd, GDEV_IOCTL_GPHYSGET, &phys);
+
+ return phys.phys;
+}
View
14 runtime/user/pscnv/libpscnv.c
@@ -335,3 +335,17 @@ int pscnv_vm_unmap(int fd, uint32_t vid, uint32_t handle) {
return ret;
return 0;
}
+
+int pscnv_phys_getaddr(int fd, uint32_t vid, uint32_t handle, uint64_t addr, uint64_t offset, uint64_t *phys) {
+ int ret;
+ struct drm_pscnv_phys_getaddr req;
+ req.vid = vid;
+ req.handle = handle;
+ req.addr = addr;
+ req.offset = offset;
+ ret = drmCommandWriteRead(fd, DRM_PSCNV_PHYS_GETADDR, &req, sizeof(req));
+ if (ret)
+ return ret;
+ *phys = req.phys;
+ return 0;
+}
View
1 runtime/user/pscnv/libpscnv.h
@@ -51,6 +51,7 @@ int pscnv_vm_read(int fd, uint32_t vid, uint32_t handle, uint64_t addr, void *bu
int pscnv_vm_write(int fd, uint32_t vid, uint32_t handle, uint64_t addr, const void *buf, uint32_t size);
int pscnv_vm_map(int fd, uint32_t vid, uint32_t handle, uint64_t *map_handle);
int pscnv_vm_unmap(int fd, uint32_t vid, uint32_t handle);
+int pscnv_phys_getaddr(int fd, uint32_t vid, uint32_t handle, uint64_t addr, uint64_t offset, uint64_t *phys);
#define pscnv_obj_gr_new pscnv_obj_eng_new
View
9 runtime/user/pscnv/pscnv_drm.h
@@ -162,6 +162,14 @@ struct drm_pscnv_vm_map {
uint64_t map_handle; /* > < */
};
+struct drm_pscnv_phys_getaddr {
+ uint32_t vid; /* < */
+ uint32_t handle; /* < */
+ uint64_t addr; /* < */
+ uint32_t offset; /* < */
+ uint64_t phys; /* > */
+};
+
#define DRM_PSCNV_GETPARAM 0x00 /* get some information from the card */
#define DRM_PSCNV_GEM_NEW 0x20 /* create a new BO */
#define DRM_PSCNV_GEM_INFO 0x21 /* get info about a BO */
@@ -182,5 +190,6 @@ struct drm_pscnv_vm_map {
#define DRM_PSCNV_VM_WRITE 0x2f /* Write to virtual memory */
#define DRM_PSCNV_VM_MAP 0x30 /* Map virtual memory */
#define DRM_PSCNV_VM_UNMAP 0x31 /* Unmap virtual memory */
+#define DRM_PSCNV_PHYS_GETADDR 0x32 /* Get physical address */
#endif /* __PSCNV_DRM_H__ */
View
23 runtime/user/pscnv/pscnv_gdev.c
@@ -402,16 +402,18 @@ void *gdev_raw_mem_map(struct gdev_mem *mem)
uint64_t map_handle;
void *map;
- pscnv_vm_map(fd, vid, handle, &map_handle);
+ if (pscnv_vm_map(fd, vid, handle, &map_handle))
+ goto fail_vm_map;
map = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, map_handle);
if (map == MAP_FAILED)
- goto fail;
+ goto fail_mmap;
return map;
-fail:
+fail_mmap:
pscnv_vm_unmap(fd, vid, handle);
+fail_vm_map:
return NULL;
}
@@ -428,3 +430,18 @@ void gdev_raw_mem_unmap(struct gdev_mem *mem, void *map)
pscnv_vm_unmap(fd, vid, handle);
}
+/* get physical bus address. */
+uint64_t gdev_raw_mem_phys_getaddr(struct gdev_mem *mem, uint64_t offset)
+{
+ struct pscnv_ib_bo *bo = mem->bo;
+ int fd = bo->fd;
+ int vid = bo->vid;
+ uint32_t handle = bo->handle;
+ uint64_t addr = mem->addr;
+ uint64_t phys;
+
+ if (pscnv_phys_getaddr(fd, vid, handle, addr, offset, &phys))
+ return 0;
+
+ return phys;
+}
View
23 test/cuda/common/vmmap.c
@@ -34,9 +34,8 @@ int cuda_test_madd(unsigned int n, char *path)
CUfunction function;
CUmodule module;
CUdeviceptr a_dev, b_dev, c_dev;
- unsigned int *a_buf;
- unsigned int *b_buf;
- unsigned int *c_buf;
+ unsigned int *a_buf, *b_buf, *c_buf;
+ unsigned long long int a_phys, b_phys, c_phys;
int block_x, block_y, grid_x, grid_y;
char fname[256];
int ret = 0;
@@ -97,6 +96,12 @@ int cuda_test_madd(unsigned int n, char *path)
printf("cuMemMap (a) failed\n");
return -1;
}
+ res = cuMemGetPhysAddr(&a_phys, (void*)a_buf);
+ if (res != CUDA_SUCCESS) {
+ printf("cuMemGetPhysAddress (a) failed\n");
+ return -1;
+ }
+ printf("a[]: Physical Address 0x%llx\n", a_phys);
/* b[] */
res = cuMemAlloc(&b_dev, n*n * sizeof(unsigned int));
@@ -109,6 +114,12 @@ int cuda_test_madd(unsigned int n, char *path)
printf("cuMemMap (b) failed\n");
return -1;
}
+ res = cuMemGetPhysAddr(&b_phys, (void*)b_buf);
+ if (res != CUDA_SUCCESS) {
+ printf("cuMemGetPhysAddress (b) failed\n");
+ return -1;
+ }
+ printf("b[]: Physical Address 0x%llx\n", b_phys);
/* c[] */
res = cuMemAlloc(&c_dev, n*n * sizeof(unsigned int));
@@ -121,6 +132,12 @@ int cuda_test_madd(unsigned int n, char *path)
printf("cuMemMap (c) failed\n");
return -1;
}
+ res = cuMemGetPhysAddr(&c_phys, (void*)c_buf);
+ if (res != CUDA_SUCCESS) {
+ printf("cuMemGetPhysAddress (c) failed\n");
+ return -1;
+ }
+ printf("c[]: Physical Address 0x%llx\n", c_phys);
/* initialize A[] & B[] */
for (i = 0; i < n; i++) {

0 comments on commit 2178fc4

Please sign in to comment.
Something went wrong with that request. Please try again.