Browse files

gdev: updated shared memory support

  • Loading branch information...
1 parent ddb9864 commit 6b4ff38cdd7c47eaad1aa3d939c97e436076e795 Shinpei Kato committed Jan 5, 2012
View
85 common/gdev_api.c
@@ -275,12 +275,12 @@ static int __gmemcpy_to_device
if (!mem)
return -ENOENT;
- gdev_shmem_lock(mem);
- gdev_shmem_evict(ctx, mem); /* evict data only if necessary */
+ gdev_shm_lock(mem);
+ gdev_shm_evict(ctx, mem); /* evict data only if necessary */
ret = __gmemcpy_to_device_locked(ctx, dst_addr, src_buf, size, async,
ch_size, p_count, vas, mem, dma_mem,
host_copy);
- gdev_shmem_unlock(mem);
+ gdev_shm_unlock(mem);
return ret;
}
@@ -463,12 +463,12 @@ static int __gmemcpy_from_device
if (!mem)
return -ENOENT;
- gdev_shmem_lock(mem);
- gdev_shmem_reload(ctx, mem); /* reload data only if necessary. */
+ gdev_shm_lock(mem);
+ gdev_shm_reload(ctx, mem); /* reload data only if necessary. */
ret = __gmemcpy_from_device_locked(ctx, dst_buf, src_addr, size, async,
ch_size, p_count, vas, mem, dma_mem,
host_copy);
- gdev_shmem_unlock(mem);
+ gdev_shm_unlock(mem);
return ret;
}
@@ -607,9 +607,6 @@ struct gdev_handle *gopen(int minor)
h->gdev = gdev;
h->dev_id = minor;
- /* insert the created VAS object to the device VAS list. */
- gdev_vas_list_add(vas);
-
GDEV_PRINT("Opened gdev%d\n", minor);
return h;
@@ -637,9 +634,6 @@ int gclose(struct gdev_handle *h)
if (!h->gdev || !h->ctx || !h->vas || !h->se)
return -ENOENT;
- /* delete the VAS object from the device VAS list. */
- gdev_vas_list_del(h->vas);
-
/* free the scheduling entity. */
gdev_sched_entity_destroy(h->se);
@@ -675,15 +669,15 @@ uint64_t gmalloc(struct gdev_handle *h, uint64_t size)
gdev->mem_used += size;
if (gdev->mem_used > gdev->mem_size) {
- /* a second chance with shared memory (only for device memory). */
- if (!(mem = gdev_shmem_request(vas, NULL, size)))
+ /* try to share memory with someone (only for device memory).
+ the shared memory must be freed in gdev_mem_free() when
+ unreferenced by all users. */
+ if (!(mem = gdev_mem_share(vas, size)))
goto fail;
}
else if (!(mem = gdev_mem_alloc(vas, size, GDEV_MEM_DEVICE)))
goto fail;
- gdev_mem_list_add(mem, GDEV_MEM_DEVICE);
-
return gdev_mem_get_addr(mem);
fail:
@@ -705,7 +699,6 @@ uint64_t gfree(struct gdev_handle *h, uint64_t addr)
if (!(mem = gdev_mem_lookup(vas, addr, GDEV_MEM_DEVICE)))
goto fail;
size = gdev_mem_get_size(mem);
- gdev_mem_list_del(mem);
gdev_mem_free(mem);
gdev->mem_used -= size;
@@ -732,7 +725,6 @@ void *gmalloc_dma(struct gdev_handle *h, uint64_t size)
goto fail;
else if (!(mem = gdev_mem_alloc(vas, size, GDEV_MEM_DMA)))
goto fail;
- gdev_mem_list_add(mem, GDEV_MEM_DMA);
return gdev_mem_get_buf(mem);
@@ -755,7 +747,6 @@ uint64_t gfree_dma(struct gdev_handle *h, void *buf)
if (!(mem = gdev_mem_lookup(vas, (uint64_t)buf, GDEV_MEM_DMA)))
goto fail;
size = gdev_mem_get_size(mem);
- gdev_mem_list_del(mem);
gdev_mem_free(mem);
gdev->dma_mem_used -= size;
@@ -767,30 +758,6 @@ uint64_t gfree_dma(struct gdev_handle *h, void *buf)
}
/**
- * gmalloc_shm():
- * allocate new device memory, if necessary, and reference it.
- */
-uint64_t gmalloc_shm(Ghandle h, int key, uint32_t flags, uint64_t size)
-{
- struct gdev_device *gdev = h->gdev;
- gdev_vas_t *vas = h->vas;
- gdev_mem_t *mem;
-
-fail:
- return 0;
-}
-
-/**
- * gfree_shm():
- * unreference the shared memory space, and free it if necessary.
- */
-uint64_t gfree_shm(Ghandle h, uint64_t addr)
-{
-fail:
- return 0;
-}
-
-/**
* gmemcpy_to_device():
* copy data from @buf to device memory at @addr.
*/
@@ -894,12 +861,12 @@ int gmemcpy_in_device
if (!dst || !src)
return -ENOENT;
- gdev_shmem_lock(dst);
- gdev_shmem_lock(src);
+ gdev_shm_lock(dst);
+ gdev_shm_lock(src);
fence = gdev_memcpy(ctx, dst_addr, src_addr, size, 0);
gdev_poll(ctx, fence, NULL);
- gdev_shmem_unlock(src);
- gdev_shmem_unlock(dst);
+ gdev_shm_unlock(src);
+ gdev_shm_unlock(dst);
return 0;
}
@@ -916,10 +883,10 @@ int glaunch(struct gdev_handle *h, struct gdev_kernel *kernel, uint32_t *id)
gdev_schedule_launch(se);
- gdev_shmem_lock_all(vas);
- gdev_shmem_reload_all(ctx, vas); /* this reloads data only if necessary */
+ gdev_shm_lock_all(vas);
+ gdev_shm_reload_all(ctx, vas); /* this reloads data only if necessary */
*id = gdev_launch(ctx, kernel);
- gdev_shmem_unlock_all(vas);
+ gdev_shm_unlock_all(vas);
return 0;
}
@@ -987,3 +954,21 @@ int gtune(struct gdev_handle *h, uint32_t type, uint32_t value)
}
return 0;
}
+
+int gshmget(Ghandle h, int key, uint64_t size, int flags)
+{
+ struct gdev_device *gdev = h->gdev;
+ gdev_vas_t *vas = h->vas;
+
+ return gdev_shm_create(gdev, vas, key, size, flags);
+}
+
+uint64_t gshmat(Ghandle h, int id, uint64_t addr, int flags)
+{
+ return 0;
+}
+
+uint64_t gshmdt(Ghandle h, uint64_t addr)
+{
+ return 0;
+}
View
6 common/gdev_api.h
@@ -48,8 +48,6 @@ uint64_t gmalloc(Ghandle h, uint64_t size);
uint64_t gfree(Ghandle h, uint64_t addr);
void *gmalloc_dma(Ghandle h, uint64_t size);
uint64_t gfree_dma(Ghandle h, void *buf);
-uint64_t gmalloc_shm(Ghandle h, int key, uint32_t flags, uint64_t size);
-uint64_t gfree_shm(Ghandle h, uint64_t addr);
int gmemcpy_to_device(Ghandle h, uint64_t dst_addr, const void *src_buf, uint64_t size);
int gmemcpy_to_device_async(Ghandle h, uint64_t dst_addr, const void *src_buf, uint64_t size);
int gmemcpy_user_to_device(Ghandle h, uint64_t dst_addr, const void *src_buf, uint64_t size);
@@ -63,6 +61,10 @@ int glaunch(Ghandle h, struct gdev_kernel *kernel, uint32_t *id);
int gsync(Ghandle h, uint32_t id, struct gdev_time *timeout);
int gquery(Ghandle h, uint32_t type, uint64_t *result);
int gtune(Ghandle h, uint32_t type, uint32_t value);
+int gshmget(Ghandle h, int key, uint64_t size, int flags);
+uint64_t gshmat(Ghandle h, int id, uint64_t addr, int flags);
+uint64_t gshmdt(Ghandle h, uint64_t addr);
+
/**
* tuning types for Gdev resource management parameters.
View
30 common/gdev_arch.h
@@ -65,26 +65,26 @@ gdev_ctx_t *gdev_ctx_new(struct gdev_device *gdev, gdev_vas_t *vas);
void gdev_ctx_free(gdev_ctx_t *ctx);
int gdev_ctx_get_cid(gdev_ctx_t *ctx);
gdev_mem_t *gdev_mem_alloc(gdev_vas_t *vas, uint64_t size, int type);
+gdev_mem_t *gdev_mem_share(gdev_vas_t *vas, uint64_t size);
void gdev_mem_free(gdev_mem_t *mem);
void gdev_mem_gc(gdev_vas_t *vas);
-int gdev_shmem_evict(gdev_ctx_t *ctx, gdev_mem_t *mem);
-int gdev_shmem_evict_all(gdev_ctx_t *ctx, gdev_vas_t *vas);
-int gdev_shmem_reload(gdev_ctx_t *ctx, gdev_mem_t *mem);
-int gdev_shmem_reload_all(gdev_ctx_t *ctx, gdev_vas_t *vas);
-gdev_mem_t *gdev_shmem_request(gdev_vas_t *vas, gdev_mem_t *mem, uint64_t size);
-void gdev_shmem_lock(gdev_mem_t *mem);
-void gdev_shmem_unlock(gdev_mem_t *mem);
-void gdev_shmem_lock_all(gdev_vas_t *vas);
-void gdev_shmem_unlock_all(gdev_vas_t *vas);
-int gdev_swap_create(struct gdev_device *gdev, uint32_t size);
-void gdev_swap_destroy(struct gdev_device *gdev);
-void gdev_vas_list_add(gdev_vas_t *vas);
-void gdev_vas_list_del(gdev_vas_t *vas);
-void gdev_mem_list_add(gdev_mem_t *mem, int type);
-void gdev_mem_list_del(gdev_mem_t *mem);
gdev_mem_t *gdev_mem_lookup(gdev_vas_t *vas, uint64_t addr, int type);
void *gdev_mem_get_buf(gdev_mem_t *mem);
uint64_t gdev_mem_get_addr(gdev_mem_t *mem);
uint64_t gdev_mem_get_size(gdev_mem_t *mem);
+int gdev_shm_create(struct gdev_device *gdev, gdev_vas_t *vas, int key, uint64_t size, int flags);
+int gdev_shm_destroy(struct gdev_device *gdev, int id);
+gdev_mem_t *gdev_shm_attach(gdev_vas_t *vas, gdev_mem_t *mem, uint64_t size);
+void gdev_shm_detach(struct gdev_mem *mem);
+int gdev_shm_evict(gdev_ctx_t *ctx, gdev_mem_t *mem);
+int gdev_shm_evict_all(gdev_ctx_t *ctx, gdev_vas_t *vas);
+int gdev_shm_reload(gdev_ctx_t *ctx, gdev_mem_t *mem);
+int gdev_shm_reload_all(gdev_ctx_t *ctx, gdev_vas_t *vas);
+void gdev_shm_lock(gdev_mem_t *mem);
+void gdev_shm_unlock(gdev_mem_t *mem);
+void gdev_shm_lock_all(gdev_vas_t *vas);
+void gdev_shm_unlock_all(gdev_vas_t *vas);
+int gdev_swap_create(struct gdev_device *gdev, uint32_t size);
+void gdev_swap_destroy(struct gdev_device *gdev);
#endif
View
6 common/gdev_device.c
@@ -51,8 +51,9 @@ int gdev_init_device(struct gdev_device *gdev, int minor, void *priv)
gdev->parent = NULL;
gdev->priv = priv; /* this must be set before calls to gdev_query(). */
gdev_list_init(&gdev->vas_list, NULL); /* VAS list. */
+ gdev_list_init(&gdev->shm_list, NULL); /* shared memory list. */
gdev_lock_init(&gdev->vas_lock);
- gdev_mutex_init(&gdev->shmem_mutex);
+ gdev_mutex_init(&gdev->shm_mutex);
/* architecture-dependent chipset.
this call must be prior to the following. */
@@ -96,8 +97,9 @@ int gdev_init_virtual_device
gdev->dma_mem_size = phys->dma_mem_size * mem_util / 100;
gdev->chipset = phys->chipset;
gdev_list_init(&gdev->vas_list, NULL); /* VAS list. */
+ gdev_list_init(&gdev->shm_list, NULL); /* shared memory list. */
gdev_lock_init(&gdev->vas_lock);
- gdev_mutex_init(&gdev->shmem_mutex);
+ gdev_mutex_init(&gdev->shm_mutex);
/* create the swap memory object, if configured, for the virtual device. */
if (GDEV_SWAP_MEM_SIZE > 0) {
View
3 common/gdev_device.h
@@ -51,8 +51,9 @@ struct gdev_device {
void *sched_thread; /* scheduler thread */
struct gdev_device *parent; /* only for virtual devices */
struct gdev_list vas_list; /* list of VASes allocated to this device */
+ struct gdev_list shm_list; /* list of VASes allocated to this device */
gdev_lock_t vas_lock;
- gdev_mutex_t shmem_mutex;
+ gdev_mutex_t shm_mutex;
gdev_mem_t *swap; /* reserved swap memory space */
};
View
841 common/gdev_nvidia.c
@@ -32,243 +32,42 @@
#include "gdev_nvidia.h"
#include "gdev_time.h"
-/**
- * these functions must be used when evicting/reloading data.
- * better implementations wanted!
- */
-int gdev_callback_evict_to_host(void*, void*, uint64_t, uint64_t);
-int gdev_callback_evict_to_device(void*, uint64_t, uint64_t, uint64_t);
-int gdev_callback_reload_from_host(void*, uint64_t, void*, uint64_t);
-int gdev_callback_reload_from_device(void*, uint64_t, uint64_t, uint64_t);
-
-/**
- * memcpy functions prototypes
- */
-static uint32_t gdev_memcpy_sync
-(struct gdev_ctx *, uint64_t, uint64_t, uint32_t, uint32_t);
-static uint32_t gdev_memcpy_async
-(struct gdev_ctx *, uint64_t, uint64_t, uint32_t, uint32_t);
-
-/**
- * pointers to memcpy functions.
- * gdev_memcpy_func[0] is synchronous memcpy.
- * gdev_memcpy_func[1] is asynchrounous memcpy.
- */
-static uint32_t (*gdev_memcpy_func[2])
-(struct gdev_ctx*, uint64_t, uint64_t, uint32_t, uint32_t) = {
- gdev_memcpy_sync,
- gdev_memcpy_async
-};
-
-/* set up the architecture-dependent compute engine. */
-int gdev_compute_setup(struct gdev_device *gdev)
-{
- switch (gdev->chipset & 0xf0) {
- case 0xC0:
- nvc0_compute_setup(gdev);
- break;
- case 0x50:
- case 0x80:
- case 0x90:
- case 0xA0:
- /* TODO: create the compute and m2mf subchannels! */
- GDEV_PRINT("NV%x not supported.\n", gdev->chipset);
- return -EINVAL;
- default:
- GDEV_PRINT("NV%x not supported.\n", gdev->chipset);
- return -EINVAL;
- }
-
- return 0;
-}
-
-/* launch the kernel onto the GPU. */
-uint32_t gdev_launch(struct gdev_ctx *ctx, struct gdev_kernel *kern)
+/* open a new Gdev object associated with the specified device. */
+struct gdev_device *gdev_dev_open(int minor)
{
- struct gdev_vas *vas = ctx->vas;
- struct gdev_device *gdev = vas->gdev;
- struct gdev_mem *dev_swap = gdev->swap;
- struct gdev_compute *compute = gdev->compute;
- uint32_t seq;
-
- /* evict data saved in device swap memory space to host memory. */
- if (dev_swap && dev_swap->shmem->holder) {
- struct gdev_mem *mem = dev_swap->shmem->holder;
- gdev_shmem_evict(ctx, mem->swap_mem); /* don't pass gdev->swap! */
- dev_swap->shmem->holder = NULL;
- }
-
- if (++ctx->fence.seq == GDEV_FENCE_COUNT)
- ctx->fence.seq = 1;
- seq = ctx->fence.seq;
-
- compute->membar(ctx);
- /* it's important to emit a fence *after* launch():
- the LAUNCH method of the PGRAPH engine is not associated with
- the QUERY method, i.e., we have to submit the QUERY method
- explicitly after the kernel is launched. */
- compute->fence_reset(ctx, seq);
- compute->launch(ctx, kern);
- compute->fence_write(ctx, GDEV_SUBCH_COMPUTE, seq);
-
- /* set an interrupt to be caused when compute done. */
- compute->notify_intr(ctx);
-
- return seq;
+ return gdev_raw_dev_open(minor);
}
-/* synchrounously copy data of @size from @src_addr to @dst_addr. */
-static uint32_t gdev_memcpy_sync
-(struct gdev_ctx *ctx, uint64_t dst_addr, uint64_t src_addr, uint32_t size,
- uint32_t seq)
+/* close the specified Gdev object. */
+void gdev_dev_close(struct gdev_device *gdev)
{
- struct gdev_vas *vas = ctx->vas;
- struct gdev_device *gdev = vas->gdev;
- struct gdev_compute *compute = gdev->compute;
-
- compute->membar(ctx);
- /* it's important to emit a fence *before* memcpy():
- the EXEC method of the PCOPY and M2MF engines is associated with
- the QUERY method, i.e., if QUERY is set, the sequence will be
- written to the specified address when the data are transfered. */
- compute->fence_reset(ctx, seq);
- compute->fence_write(ctx, GDEV_SUBCH_M2MF, seq);
- compute->memcpy(ctx, dst_addr, src_addr, size);
-
- return seq;
+ gdev_raw_dev_close(gdev);
}
-/* asynchrounously copy data of @size from @src_addr to @dst_addr. */
-static uint32_t gdev_memcpy_async
-(struct gdev_ctx *ctx, uint64_t dst_addr, uint64_t src_addr, uint32_t size,
- uint32_t seq)
+/* add a new VAS object into the device VAS list. */
+static void __gdev_vas_list_add(struct gdev_vas *vas)
{
- struct gdev_vas *vas = ctx->vas;
struct gdev_device *gdev = vas->gdev;
- struct gdev_compute *compute = gdev->compute;
-
- compute->membar(ctx);
- /* it's important to emit a fence *before* memcpy():
- the EXEC method of the PCOPY and M2MF engines is associated with
- the QUERY method, i.e., if QUERY is set, the sequence will be
- written to the specified address when the data are transfered. */
- compute->fence_reset(ctx, seq);
- compute->fence_write(ctx, GDEV_SUBCH_PCOPY0, seq);
- compute->memcpy_async(ctx, dst_addr, src_addr, size);
-
- return seq;
-}
-
-/* asynchrounously copy data of @size from @src_addr to @dst_addr. */
-uint32_t gdev_memcpy
-(struct gdev_ctx *ctx, uint64_t dst_addr, uint64_t src_addr, uint32_t size,
- int async)
-{
- uint32_t seq;
-
- if (++ctx->fence.seq == GDEV_FENCE_COUNT)
- ctx->fence.seq = 1;
- seq = ctx->fence.seq;
-
- return gdev_memcpy_func[async](ctx, dst_addr, src_addr, size, seq);
-}
-
-/* read 32-bit value from @addr. */
-uint32_t gdev_read32(struct gdev_mem *mem, uint64_t addr)
-{
- return gdev_raw_read32(mem, addr);
-}
-
-/* write 32-bit @val to @addr. */
-void gdev_write32(struct gdev_mem *mem, uint64_t addr, uint32_t val)
-{
- gdev_raw_write32(mem, addr, val);
-}
-
-/* read @size of data from @addr. */
-int gdev_read(struct gdev_mem *mem, void *buf, uint64_t addr, uint32_t size)
-{
- return gdev_raw_read(mem, buf, addr, size);
-}
-
-/* write @size of data to @addr. */
-int gdev_write(struct gdev_mem *mem, uint64_t addr, const void *buf, uint32_t size)
-{
- return gdev_raw_write(mem, addr, buf, size);
+ unsigned long flags;
+
+ gdev_lock_save(&gdev->vas_lock, &flags);
+ gdev_list_add(&vas->list_entry, &gdev->vas_list);
+ gdev_unlock_restore(&gdev->vas_lock, &flags);
}
-/* poll until the resource becomes available. */
-int gdev_poll(struct gdev_ctx *ctx, uint32_t seq, struct gdev_time *timeout)
+/* delete the VAS object from the device VAS list. */
+static void __gdev_vas_list_del(struct gdev_vas *vas)
{
- struct gdev_time time_start, time_now, time_elapse, time_relax;
- struct gdev_vas *vas = ctx->vas;
struct gdev_device *gdev = vas->gdev;
- struct gdev_compute *compute = gdev->compute;
-
- gdev_time_stamp(&time_start);
- gdev_time_ms(&time_relax, 100); /* relax polling when 100 ms elapsed. */
-
- while (seq != compute->fence_read(ctx, seq)) {
- gdev_time_stamp(&time_now);
- /* time_elapse = time_now - time_start */
- gdev_time_sub(&time_elapse, &time_now, &time_start);
- /* relax polling after some time. */
- if (gdev_time_ge(&time_elapse, &time_relax))
- SCHED_YIELD();
- /* check timeout. */
- if (timeout && gdev_time_ge(&time_elapse, timeout))
- return -ETIME;
- }
-
- compute->fence_reset(ctx, seq);
-
- return 0;
-}
-
-/* query device-specific information. */
-int gdev_query(struct gdev_device *gdev, uint32_t type, uint64_t *result)
-{
- int ret;
-
- switch (type) {
- case GDEV_QUERY_DEVICE_MEM_SIZE:
- if (gdev->mem_size)
- *result = gdev->mem_size;
- else if ((ret = gdev_raw_query(gdev, type, result)))
- return ret;
- break;
- case GDEV_QUERY_DMA_MEM_SIZE:
- if (gdev->dma_mem_size)
- *result = gdev->dma_mem_size;
- /* FIXME: this is valid only for PCIE. */
- else if (gdev->chipset > 0x40)
- *result = 512 * 1024 * 1024;
- else
- *result = 64 * 1024 * 1024;
- break;
- default:
- if ((ret = gdev_raw_query(gdev, type, result)))
- return ret;
- }
-
- return 0;
-}
-
-/* open a new Gdev object associated with the specified device. */
-struct gdev_device *gdev_dev_open(int minor)
-{
- return gdev_raw_dev_open(minor);
-}
-
-/* close the specified Gdev object. */
-void gdev_dev_close(struct gdev_device *gdev)
-{
- gdev_raw_dev_close(gdev);
+ unsigned long flags;
+
+ gdev_lock_save(&gdev->vas_lock, &flags);
+ gdev_list_del(&vas->list_entry);
+ gdev_unlock_restore(&gdev->vas_lock, &flags);
}
/* allocate a new virual address space (VAS) object. */
-struct gdev_vas *gdev_vas_new
-(struct gdev_device *gdev, uint64_t size, void *handle)
+struct gdev_vas *gdev_vas_new(struct gdev_device *gdev, uint64_t size, void *handle)
{
struct gdev_vas *vas;
@@ -284,12 +83,15 @@ struct gdev_vas *gdev_vas_new
gdev_list_init(&vas->dma_mem_list, NULL); /* host dma memory list. */
gdev_lock_init(&vas->lock);
+ __gdev_vas_list_add(vas);
+
return vas;
}
/* free the specified virtual address space object. */
void gdev_vas_free(struct gdev_vas *vas)
{
+ __gdev_vas_list_del(vas);
gdev_raw_vas_free(vas);
}
@@ -322,598 +124,3 @@ int gdev_ctx_get_cid(struct gdev_ctx *ctx)
{
return ctx->cid;
}
-
-/* initialize a memory object. */
-static void __gdev_mem_init
-(struct gdev_mem *mem, struct gdev_vas *vas, uint64_t addr, uint64_t size,
- void *map, int type)
-{
- mem->vas = vas;
- mem->addr = addr;
- mem->size = size;
- mem->map = map;
- mem->type = type;
- mem->evicted = 0;
- mem->swap_mem = NULL;
- mem->swap_buf = NULL;
- mem->shmem = NULL;
-
- gdev_list_init(&mem->list_entry_heap, (void *) mem);
- gdev_list_init(&mem->list_entry_shmem, (void *) mem);
-}
-
-/* attach device memory and allocate host buffer for swap */
-static int __gdev_swap_create(struct gdev_mem *mem)
-{
- struct gdev_vas *vas = mem->vas;
- struct gdev_device *gdev = vas->gdev;
- struct gdev_mem *swap_mem;
- uint64_t addr, size;
- void *map, *swap_buf;
-
- /* host buffer for swap. */
- swap_buf = MALLOC(mem->size);
- if (!swap_buf)
- goto fail_swap_buf;
- /* device memory for temporal swap (shared by others). */
- if (GDEV_SWAP_MEM_SIZE > 0) {
- swap_mem = gdev_raw_mem_share(vas, gdev->swap, &addr, &size, &map);
- if (!swap_mem)
- goto fail_swap_mem;
- __gdev_mem_init(swap_mem, vas, addr, size, map, GDEV_MEM_DEVICE);
- }
- else {
- swap_mem = NULL;
- }
- mem->swap_buf = swap_buf;
- mem->swap_mem = swap_mem;
-
- return 0;
-
-fail_swap_mem:
- FREE(swap_buf);
-fail_swap_buf:
- return -ENOMEM;
-}
-
-/* dettach device memory and free host buffer for swap */
-static void __gdev_swap_destroy(struct gdev_mem *mem)
-{
- if (GDEV_SWAP_MEM_SIZE > 0)
- gdev_raw_mem_unshare(mem->swap_mem);
- FREE(mem->swap_buf);
-}
-
-/* allocate a new memory object. */
-struct gdev_mem *gdev_mem_alloc(struct gdev_vas *vas, uint64_t size, int type)
-{
- struct gdev_mem *mem;
- uint64_t addr;
- void *map;
-
- switch (type) {
- case GDEV_MEM_DEVICE:
- if (!(mem = gdev_raw_mem_alloc(vas, &addr, &size, &map)))
- goto fail;
- break;
- case GDEV_MEM_DMA:
- if (!(mem = gdev_raw_mem_alloc_dma(vas, &addr, &size, &map)))
- goto fail;
- break;
- default:
- GDEV_PRINT("Memory type not supported\n");
- goto fail;
- }
-
- __gdev_mem_init(mem, vas, addr, size, map, type);
- return mem;
-
-fail:
- return NULL;
-}
-
-/* free the specified memory object. */
-void gdev_mem_free(struct gdev_mem *mem)
-{
- struct gdev_vas *vas = mem->vas;
- struct gdev_device *gdev = vas->gdev;
- struct gdev_shmem *shmem = mem->shmem;
- struct gdev_mem *m;
-
- gdev_mutex_lock(&gdev->shmem_mutex); /* be careful of dead lock. */
-
- /* if the memory object is not shared, free it. */
- if (!shmem) {
- gdev_raw_mem_free(mem);
- }
- else {
- gdev_mutex_lock(&shmem->mutex);
- mem->shmem = NULL;
- gdev_list_del(&mem->list_entry_shmem);
- __gdev_swap_destroy(mem);
- /* if the memory object is shared but no users, free it.
- since users == 0, no one else will use mem->shmem. */
- if (shmem->users == 0) {
- gdev_raw_mem_free(mem);
- gdev_mutex_unlock(&shmem->mutex);
- FREE(shmem);
- }
- /* otherwise, just unshare the memory object. */
- else {
- /* if a freeing memory object has the highest priority among the
- shared memory objects, find the next highest priority one. */
- if (shmem->prio == vas->prio) {
- int prio = GDEV_PRIO_MIN;
- gdev_list_for_each (m, &shmem->shmem_list, list_entry_shmem) {
- if (m->vas->prio > prio)
- prio = m->vas->prio;
- }
- shmem->prio = prio;
- }
- if (shmem->holder == mem)
- shmem->holder = NULL;
- /* unshare the memory object. */
- gdev_raw_mem_unshare(mem);
- shmem->users--;
- gdev_mutex_unlock(&shmem->mutex);
- }
- }
-
- gdev_mutex_unlock(&gdev->shmem_mutex);
-}
-
-/* garbage collection: free all memory left in heap. */
-void gdev_mem_gc(struct gdev_vas *vas)
-{
- struct gdev_mem *mem;
-
- /* device memory. */
- gdev_list_for_each (mem, &vas->mem_list, list_entry_heap) {
- gdev_mem_free(mem);
- }
-
- /* host DMA memory. */
- gdev_list_for_each (mem, &vas->dma_mem_list, list_entry_heap) {
- gdev_mem_free(mem);
- }
-}
-
-/* evict the shared memory object data.
- the shared memory object associated with @mem must be locked. */
-int gdev_shmem_evict(struct gdev_ctx *ctx, struct gdev_mem *mem)
-{
- struct gdev_vas *vas = mem->vas;
- struct gdev_device *gdev = vas->gdev;
- struct gdev_mem *holder;
- uint64_t src_addr;
- uint64_t size;
- int ret;
-
- if (mem->shmem) {
- void *h = ctx->vas->handle;
- if (mem->shmem->holder && mem->shmem->holder != mem) {
- struct gdev_mem *dev_swap = gdev->swap;
- holder = mem->shmem->holder;
- src_addr = mem->addr;
- size = mem->shmem->size;
- if (dev_swap && !dev_swap->shmem->holder) {
- uint64_t dst_addr = holder->swap_mem->addr;
- ret = gdev_callback_evict_to_device(h,dst_addr,src_addr,size);
- if (ret)
- goto fail_evict;
- dev_swap->shmem->holder = mem;
- }
- else {
- void *dst_buf = holder->swap_buf;
- ret = gdev_callback_evict_to_host(h,dst_buf,src_addr,size);
- if (ret)
- goto fail_evict;
- }
- holder->evicted = 1;
- }
- mem->shmem->holder = mem;
- }
-
- return 0;
-
-
-fail_evict:
- return ret;
-}
-
-/* evict all the shared memory object data associated to @vas.
- all the shared memory objects associated to @vas must be locked. */
-int gdev_shmem_evict_all(struct gdev_ctx *ctx, struct gdev_vas *vas)
-{
- struct gdev_mem *mem;
-
- gdev_list_for_each (mem, &vas->mem_list, list_entry_heap) {
- gdev_shmem_evict(ctx, mem);
- }
-
- return 0;
-}
-
-/* reload the evicted memory object data.
- the shared memory object associated with @mem must be locked. */
-int gdev_shmem_reload(struct gdev_ctx *ctx, struct gdev_mem *mem)
-{
- struct gdev_vas *vas = ctx->vas;
- struct gdev_device *gdev = vas->gdev;
- struct gdev_mem *dev_swap;
- uint64_t dst_addr;
- uint64_t size;
- int ret;
-
- if (mem->evicted) {
- void *h = vas->handle;
- /* evict the corresponding memory space first. */
- gdev_shmem_evict(ctx, mem);
- /* reload data regardless whether eviction succeeded or failed. */
- dev_swap = gdev->swap;
- dst_addr = mem->addr;
- size = mem->size;
- if (dev_swap && dev_swap->shmem->holder == mem) {
- uint64_t src_addr = mem->swap_mem->addr;
- ret = gdev_callback_reload_from_device(h, dst_addr, src_addr, size);
- if (ret)
- goto fail_reload;
- }
- else {
- void *src_buf = mem->swap_buf;
- ret = gdev_callback_reload_from_host(h, dst_addr, src_buf, size);
- if (ret)
- goto fail_reload;
- }
- mem->evicted = 0;
- mem->shmem->holder = mem;
- }
-
- return 0;
-
-fail_reload:
- return ret;
-}
-
-/* reload all the evicted memory object data associated to @vas.
- all the shared memory objects associated to @vas must be locked. */
-int gdev_shmem_reload_all(struct gdev_ctx *ctx, struct gdev_vas *vas)
-{
- struct gdev_mem *mem;
-
- gdev_list_for_each (mem, &vas->mem_list, list_entry_heap) {
- gdev_shmem_reload(ctx, mem);
- }
-
- return 0;
-}
-
-/* find a memory object that we can borrow some memory space from. */
-static struct gdev_mem *__gdev_shmem_find_victim
-(struct gdev_vas *vas, uint64_t size)
-{
- struct gdev_device *gdev = vas->gdev;
- struct gdev_vas *v;
- struct gdev_mem *m, *victim = NULL;
- struct gdev_shmem *shmem;
- unsigned long flags;
-
- if (!(shmem = MALLOC(sizeof(*shmem))))
- goto fail_shmem;
-
- /* be careful of dead lock. this lock is valid only for shared memory. */
- gdev_mutex_lock(&gdev->shmem_mutex);
-
- /* select the lowest-priority object. */
- gdev_lock_save(&gdev->vas_lock, &flags);
- gdev_list_for_each (v, &gdev->vas_list, list_entry) {
- gdev_lock_nested(&v->lock);
- gdev_list_for_each (m, &v->mem_list, list_entry_heap) {
- /* don't select from the save VAS object! */
- if (m->size >= size && m->vas != vas) {
- if (!victim)
- victim = m;
- else {
- int victim_prio, m_prio;
- if (victim->shmem)
- victim_prio = victim->shmem->prio;
- else
- victim_prio = victim->vas->prio;
- if (m->shmem)
- m_prio = m->shmem->prio;
- else
- m_prio = m->vas->prio;
- if (victim_prio > m_prio)
- victim = m;
- }
- }
- }
- gdev_unlock_nested(&v->lock);
- }
- gdev_unlock_restore(&gdev->vas_lock, &flags);
-
- if (!victim) {
- FREE(shmem);
- }
- /* if the victim object doesn't have a shared memory object yet,
- allocate a new one here. */
- else if (!victim->shmem) {
- gdev_list_init(&shmem->shmem_list, NULL);
- gdev_mutex_init(&shmem->mutex);
- shmem->prio = GDEV_PRIO_MIN;
- shmem->users = 0;
- shmem->holder = NULL;
- shmem->bo = victim->bo;
- shmem->size = victim->size;
- /* create swap for evicting data from shared memory space. */
- if (__gdev_swap_create(victim))
- goto fail_swap;
- gdev_mutex_lock(&shmem->mutex);
- /* the victim object itself must be inserted into the list. */
- gdev_list_add(&victim->list_entry_shmem, &shmem->shmem_list);
- /* someone will be using this shared memory, that's why this
- function was called. */
- shmem->users++;
- victim->shmem = shmem;
- gdev_mutex_unlock(&shmem->mutex);
- }
- else {
- gdev_mutex_lock(&victim->shmem->mutex);
- victim->shmem->users++;
- gdev_mutex_unlock(&victim->shmem->mutex);
- FREE(shmem); /* shmem was unused. */
- }
-
- gdev_mutex_unlock(&gdev->shmem_mutex);
-
- return victim;
-
-fail_swap:
- FREE(shmem);
- gdev_mutex_unlock(&gdev->shmem_mutex);
-fail_shmem:
- return NULL;
-}
-
-/* share memory space with @mem. if @mem is null, find a victim instead. */
-struct gdev_mem *gdev_shmem_request
-(struct gdev_vas *vas, struct gdev_mem *mem, uint64_t size)
-{
- struct gdev_mem *new;
- uint64_t addr;
- void *map;
-
- if (!mem) {
- /* select a victim memory object. victim->shmem will be newly
- allocated if NULL, with shmem->users being incremented. */
- if (!(mem = __gdev_shmem_find_victim(vas, size)))
- goto fail_victim;
- }
-
- /* borrow the same (physical) memory space by sharing. */
- if (!(new = gdev_raw_mem_share(vas, mem, &addr, &size, &map)))
- goto fail_shmem;
-
- /* initialize the new memory object. */
- __gdev_mem_init(new, vas, addr, size, map, GDEV_MEM_DEVICE);
-
- /* create swap. */
- if (__gdev_swap_create(new))
- goto fail_swap;
-
- gdev_mutex_lock(&mem->shmem->mutex);
- new->shmem = mem->shmem;
- /* set the highest prio among users to the shared memory's priority. */
- if (new->shmem->prio < vas->prio)
- new->shmem->prio = vas->prio;
- /* insert the new memory object into the shared memory list. */
- gdev_list_add(&new->list_entry_shmem, &new->shmem->shmem_list);
- gdev_mutex_unlock(&mem->shmem->mutex);
-
- return new;
-
-fail_swap:
- gdev_raw_mem_unshare(new);
-fail_shmem:
- mem->shmem->users--; /* not really needed? */
-fail_victim:
- return NULL;
-}
-
-/* lock the shared memory associated with @mem, if any. */
-void gdev_shmem_lock(struct gdev_mem *mem)
-{
- if (mem->shmem) {
- gdev_mutex_lock(&mem->shmem->mutex);
- }
-}
-
-/* unlock the shared memory associated with @mem, if any. */
-void gdev_shmem_unlock(struct gdev_mem *mem)
-{
- if (mem->shmem) {
- gdev_mutex_unlock(&mem->shmem->mutex);
- }
-}
-
-/* lock all the shared memory objects associated with @vas. */
-void gdev_shmem_lock_all(struct gdev_vas *vas)
-{
- struct gdev_device *gdev = vas->gdev;
- struct gdev_mem *mem;
-
- gdev_mutex_lock(&gdev->shmem_mutex);
- gdev_list_for_each (mem, &vas->mem_list, list_entry_heap) {
- gdev_shmem_lock(mem);
- }
- gdev_mutex_unlock(&gdev->shmem_mutex);
-}
-
-/* unlock all the shared memory objects associated with @vas. */
-void gdev_shmem_unlock_all(struct gdev_vas *vas)
-{
- struct gdev_device *gdev = vas->gdev;
- struct gdev_mem *mem;
-
- gdev_mutex_lock(&gdev->shmem_mutex);
- gdev_list_for_each (mem, &vas->mem_list, list_entry_heap) {
- gdev_shmem_unlock(mem);
- }
- gdev_mutex_unlock(&gdev->shmem_mutex);
-}
-
-/* create swap memory object for the device. */
-int gdev_swap_create(struct gdev_device *gdev, uint32_t size)
-{
- struct gdev_mem *swap;
- struct gdev_shmem *shmem;
- int ret = 0;
-
- swap = gdev_raw_swap_alloc(gdev, size);
- if (swap) {
- shmem = MALLOC(sizeof(*shmem));
- if (shmem)
- shmem->holder = NULL;
- else {
- gdev_raw_swap_free(swap);
- swap = NULL;
- ret = -ENOMEM;
- }
- swap->shmem = shmem;
- }
- else {
- ret = -ENOMEM;
- }
-
- gdev->swap = swap;
-
- return ret;
-}
-
-/* remove swap memory object from the device. */
-void gdev_swap_destroy(struct gdev_device *gdev)
-{
- if (gdev->swap) {
- if (gdev->swap->shmem)
- FREE(gdev->swap->shmem);
- gdev_raw_swap_free(gdev->swap);
- }
-}
-
-/* add a new VAS object into the device VAS list. */
-void gdev_vas_list_add(struct gdev_vas *vas)
-{
- struct gdev_device *gdev = vas->gdev;
- unsigned long flags;
-
- gdev_lock_save(&gdev->vas_lock, &flags);
- gdev_list_add(&vas->list_entry, &gdev->vas_list);
- gdev_unlock_restore(&gdev->vas_lock, &flags);
-}
-
-/* delete the VAS object from the device VAS list. */
-void gdev_vas_list_del(struct gdev_vas *vas)
-{
- struct gdev_device *gdev = vas->gdev;
- unsigned long flags;
-
- gdev_lock_save(&gdev->vas_lock, &flags);
- gdev_list_del(&vas->list_entry);
- gdev_unlock_restore(&gdev->vas_lock, &flags);
-}
-
-/* add a new memory object to the memory list. */
-void gdev_mem_list_add(struct gdev_mem *mem, int type)
-{
- struct gdev_vas *vas = mem->vas;
- unsigned long flags;
-
- switch (type) {
- case GDEV_MEM_DEVICE:
- gdev_lock_save(&vas->lock, &flags);
- gdev_list_add(&mem->list_entry_heap, &vas->mem_list);
- gdev_unlock_restore(&vas->lock, &flags);
- break;
- case GDEV_MEM_DMA:
- gdev_lock_save(&vas->lock, &flags);
- gdev_list_add(&mem->list_entry_heap, &vas->dma_mem_list);
- gdev_unlock_restore(&vas->lock, &flags);
- break;
- default:
- GDEV_PRINT("Memory type not supported\n");
- }
-}
-
-/* delete the memory object from the memory list. */
-void gdev_mem_list_del(struct gdev_mem *mem)
-{
- struct gdev_vas *vas = mem->vas;
- unsigned long flags;
- int type = mem->type;
-
- switch (type) {
- case GDEV_MEM_DEVICE:
- gdev_lock_save(&vas->lock, &flags);
- gdev_list_del(&mem->list_entry_heap);
- gdev_unlock_restore(&vas->lock, &flags);
- break;
- case GDEV_MEM_DMA:
- gdev_lock_save(&vas->lock, &flags);
- gdev_list_del(&mem->list_entry_heap);
- gdev_unlock_restore(&vas->lock, &flags);
- break;
- default:
- GDEV_PRINT("Memory type not supported\n");
- }
-}
-
-/* look up the memory object allocated at the specified address. */
-struct gdev_mem *gdev_mem_lookup(struct gdev_vas *vas, uint64_t addr, int type)
-{
- struct gdev_mem *mem = NULL;
- unsigned long flags;
-
- switch (type) {
- case GDEV_MEM_DEVICE:
- gdev_lock_save(&vas->lock, &flags);
- gdev_list_for_each (mem, &vas->mem_list, list_entry_heap) {
- if (mem && (addr >= mem->addr && addr < mem->addr + mem->size))
- break;
- }
- gdev_unlock_restore(&vas->lock, &flags);
- break;
- case GDEV_MEM_DMA:
- gdev_lock_save(&vas->lock, &flags);
- gdev_list_for_each (mem, &vas->dma_mem_list, list_entry_heap) {
- if (mem && (addr >= (uint64_t) mem->map &&
- addr < (uint64_t) mem->map + mem->size))
- break;
- }
- gdev_unlock_restore(&vas->lock, &flags);
- break;
- default:
- GDEV_PRINT("Memory type not supported\n");
- }
-
- return mem;
-}
-
-/* get host DMA buffer. */
-void *gdev_mem_get_buf(struct gdev_mem *mem)
-{
- if (mem->type != GDEV_MEM_DMA)
- return NULL;
- return mem->map;
-}
-
-/* get virtual memory address. */
-uint64_t gdev_mem_get_addr(struct gdev_mem *mem)
-{
- return mem->addr;
-}
-
-/* get allocated memory size. */
-uint64_t gdev_mem_get_size(struct gdev_mem *mem)
-{
- return mem->size;
-}
View
32 common/gdev_nvidia.h
@@ -60,13 +60,16 @@
/**
* Gdev shared memory information:
*/
-struct gdev_shmem {
+struct gdev_shm {
struct gdev_mem *holder; /* current memory holder */
- struct gdev_list shmem_list; /* list of shared memory users */
+ struct gdev_list mem_list; /* list of memory objects attached */
+ struct gdev_list list_entry; /* entry to the list of shared memory */
gdev_mutex_t mutex;
uint64_t size;
int prio; /* highest prio among users (effective only for master) */
int users; /* number of users (effective only for master) */
+ int key; /* key value of this shared memory */
+ int id; /* indentifier of this shared memory */
void *bo; /* private buffer object */
};
@@ -151,8 +154,8 @@ struct gdev_mem {
void *bo; /* driver private object */
struct gdev_vas *vas; /* mem is associated with a specific vas object */
struct gdev_list list_entry_heap; /* entry to heap list */
- struct gdev_list list_entry_shmem; /* entry to shared memory list */
- struct gdev_shmem *shmem; /* shared memory information */
+ struct gdev_list list_entry_shm; /* entry to shared memory list */
+ struct gdev_shm *shm; /* shared memory information */
struct gdev_mem *swap_mem; /* device memory for temporal swap */
void *swap_buf; /* host buffer for swap */
int evicted; /* 1 if evicted, 0 otherwise */
@@ -205,7 +208,26 @@ int gdev_raw_read(struct gdev_mem *mem, void *buf, uint64_t addr, uint32_t size)
int gdev_raw_write(struct gdev_mem *mem, uint64_t addr, const void *buf, uint32_t size);
/**
- * runtime/driver/architecture-independent inline FIFO functions.
+ * initialize a memory object.
+ */
+static inline void __gdev_mem_init(struct gdev_mem *mem, struct gdev_vas *vas, uint64_t addr, uint64_t size, void *map, int type)
+{
+ mem->vas = vas;
+ mem->addr = addr;
+ mem->size = size;
+ mem->map = map;
+ mem->type = type;
+ mem->evicted = 0;
+ mem->swap_mem = NULL;
+ mem->swap_buf = NULL;
+ mem->shm = NULL;
+
+ gdev_list_init(&mem->list_entry_heap, (void *) mem);
+ gdev_list_init(&mem->list_entry_shm, (void *) mem);
+}
+
+/**
+ * FIFO control functions.
*/
static inline void __gdev_relax_fifo(void)
{
View
243 common/gdev_nvidia_compute.c
@@ -0,0 +1,243 @@
+/*
+ * Copyright 2011 Shinpei Kato
+ *
+ * University of California, Santa Cruz
+ * Systems Research Lab.
+ *
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "gdev_api.h"
+#include "gdev_device.h"
+
+/**
+ * memcpy functions prototypes
+ */
+static uint32_t gdev_memcpy_sync
+(struct gdev_ctx *, uint64_t, uint64_t, uint32_t, uint32_t);
+static uint32_t gdev_memcpy_async
+(struct gdev_ctx *, uint64_t, uint64_t, uint32_t, uint32_t);
+
+/**
+ * pointers to memcpy functions.
+ * gdev_memcpy_func[0] is synchronous memcpy.
+ * gdev_memcpy_func[1] is asynchrounous memcpy.
+ */
+static uint32_t (*gdev_memcpy_func[2])
+(struct gdev_ctx*, uint64_t, uint64_t, uint32_t, uint32_t) = {
+ gdev_memcpy_sync,
+ gdev_memcpy_async
+};
+
+/* set up the architecture-dependent compute engine. */
+int gdev_compute_setup(struct gdev_device *gdev)
+{
+ switch (gdev->chipset & 0xf0) {
+ case 0xC0:
+ nvc0_compute_setup(gdev);
+ break;
+ case 0x50:
+ case 0x80:
+ case 0x90:
+ case 0xA0:
+ /* TODO: create the compute and m2mf subchannels! */
+ GDEV_PRINT("NV%x not supported.\n", gdev->chipset);
+ return -EINVAL;
+ default:
+ GDEV_PRINT("NV%x not supported.\n", gdev->chipset);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+/* launch the kernel onto the GPU. */
+uint32_t gdev_launch(struct gdev_ctx *ctx, struct gdev_kernel *kern)
+{
+ struct gdev_vas *vas = ctx->vas;
+ struct gdev_device *gdev = vas->gdev;
+ struct gdev_mem *dev_swap = gdev->swap;
+ struct gdev_compute *compute = gdev->compute;
+ uint32_t seq;
+
+ /* evict data saved in device swap memory space to host memory. */
+ if (dev_swap && dev_swap->shm->holder) {
+ struct gdev_mem *mem = dev_swap->shm->holder;
+ gdev_shm_evict(ctx, mem->swap_mem); /* don't pass gdev->swap! */
+ dev_swap->shm->holder = NULL;
+ }
+
+ if (++ctx->fence.seq == GDEV_FENCE_COUNT)
+ ctx->fence.seq = 1;
+ seq = ctx->fence.seq;
+
+ compute->membar(ctx);
+ /* it's important to emit a fence *after* launch():
+ the LAUNCH method of the PGRAPH engine is not associated with
+ the QUERY method, i.e., we have to submit the QUERY method
+ explicitly after the kernel is launched. */
+ compute->fence_reset(ctx, seq);
+ compute->launch(ctx, kern);
+ compute->fence_write(ctx, GDEV_SUBCH_COMPUTE, seq);
+
+ /* set an interrupt to be caused when compute done. */
+ compute->notify_intr(ctx);
+
+ return seq;
+}
+
+/* synchrounously copy data of @size from @src_addr to @dst_addr. */
+static uint32_t gdev_memcpy_sync
+(struct gdev_ctx *ctx, uint64_t dst_addr, uint64_t src_addr, uint32_t size,
+ uint32_t seq)
+{
+ struct gdev_vas *vas = ctx->vas;
+ struct gdev_device *gdev = vas->gdev;
+ struct gdev_compute *compute = gdev->compute;
+
+ compute->membar(ctx);
+ /* it's important to emit a fence *before* memcpy():
+ the EXEC method of the PCOPY and M2MF engines is associated with
+ the QUERY method, i.e., if QUERY is set, the sequence will be
+ written to the specified address when the data are transfered. */
+ compute->fence_reset(ctx, seq);
+ compute->fence_write(ctx, GDEV_SUBCH_M2MF, seq);
+ compute->memcpy(ctx, dst_addr, src_addr, size);
+
+ return seq;
+}
+
+/* asynchrounously copy data of @size from @src_addr to @dst_addr. */
+static uint32_t gdev_memcpy_async
+(struct gdev_ctx *ctx, uint64_t dst_addr, uint64_t src_addr, uint32_t size,
+ uint32_t seq)
+{
+ struct gdev_vas *vas = ctx->vas;
+ struct gdev_device *gdev = vas->gdev;
+ struct gdev_compute *compute = gdev->compute;
+
+ compute->membar(ctx);
+ /* it's important to emit a fence *before* memcpy():
+ the EXEC method of the PCOPY and M2MF engines is associated with
+ the QUERY method, i.e., if QUERY is set, the sequence will be
+ written to the specified address when the data are transfered. */
+ compute->fence_reset(ctx, seq);
+ compute->fence_write(ctx, GDEV_SUBCH_PCOPY0, seq);
+ compute->memcpy_async(ctx, dst_addr, src_addr, size);
+
+ return seq;
+}
+
+/* asynchrounously copy data of @size from @src_addr to @dst_addr. */
+uint32_t gdev_memcpy
+(struct gdev_ctx *ctx, uint64_t dst_addr, uint64_t src_addr, uint32_t size,
+ int async)
+{
+ uint32_t seq;
+
+ if (++ctx->fence.seq == GDEV_FENCE_COUNT)
+ ctx->fence.seq = 1;
+ seq = ctx->fence.seq;
+
+ return gdev_memcpy_func[async](ctx, dst_addr, src_addr, size, seq);
+}
+
+/* read 32-bit value from @addr. */
+uint32_t gdev_read32(struct gdev_mem *mem, uint64_t addr)
+{
+ return gdev_raw_read32(mem, addr);
+}
+
+/* write 32-bit @val to @addr. */
+void gdev_write32(struct gdev_mem *mem, uint64_t addr, uint32_t val)
+{
+ gdev_raw_write32(mem, addr, val);
+}
+
+/* read @size of data from @addr. */
+int gdev_read(struct gdev_mem *mem, void *buf, uint64_t addr, uint32_t size)
+{
+ return gdev_raw_read(mem, buf, addr, size);
+}
+
+/* write @size of data to @addr. */
+int gdev_write(struct gdev_mem *mem, uint64_t addr, const void *buf, uint32_t size)
+{
+ return gdev_raw_write(mem, addr, buf, size);
+}
+
+/* poll until the resource becomes available. */
+int gdev_poll(struct gdev_ctx *ctx, uint32_t seq, struct gdev_time *timeout)
+{
+ struct gdev_time time_start, time_now, time_elapse, time_relax;
+ struct gdev_vas *vas = ctx->vas;
+ struct gdev_device *gdev = vas->gdev;
+ struct gdev_compute *compute = gdev->compute;
+
+ gdev_time_stamp(&time_start);
+ gdev_time_ms(&time_relax, 100); /* relax polling when 100 ms elapsed. */
+
+ while (seq != compute->fence_read(ctx, seq)) {
+ gdev_time_stamp(&time_now);
+ /* time_elapse = time_now - time_start */
+ gdev_time_sub(&time_elapse, &time_now, &time_start);
+ /* relax polling after some time. */
+ if (gdev_time_ge(&time_elapse, &time_relax))
+ SCHED_YIELD();
+ /* check timeout. */
+ if (timeout && gdev_time_ge(&time_elapse, timeout))
+ return -ETIME;
+ }
+
+ compute->fence_reset(ctx, seq);
+
+ return 0;
+}
+
+/* query device-specific information. */
+int gdev_query(struct gdev_device *gdev, uint32_t type, uint64_t *result)
+{
+ int ret;
+
+ switch (type) {
+ case GDEV_QUERY_DEVICE_MEM_SIZE:
+ if (gdev->mem_size)
+ *result = gdev->mem_size;
+ else if ((ret = gdev_raw_query(gdev, type, result)))
+ return ret;
+ break;
+ case GDEV_QUERY_DMA_MEM_SIZE:
+ if (gdev->dma_mem_size)
+ *result = gdev->dma_mem_size;
+ /* FIXME: this is valid only for PCIE. */
+ else if (gdev->chipset > 0x40)
+ *result = 512 * 1024 * 1024;
+ else
+ *result = 64 * 1024 * 1024;
+ break;
+ default:
+ if ((ret = gdev_raw_query(gdev, type, result)))
+ return ret;
+ }
+
+ return 0;
+}
View
213 common/gdev_nvidia_mem.c
@@ -0,0 +1,213 @@
+/*
+ * Copyright 2011 Shinpei Kato
+ *
+ * University of California, Santa Cruz
+ * Systems Research Lab.
+ *
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "gdev_device.h"
+
+/* add a new memory object to the memory list. */
+static void __gdev_mem_list_add(struct gdev_mem *mem, int type)
+{
+ struct gdev_vas *vas = mem->vas;
+ unsigned long flags;
+
+ switch (type) {
+ case GDEV_MEM_DEVICE:
+ gdev_lock_save(&vas->lock, &flags);
+ gdev_list_add(&mem->list_entry_heap, &vas->mem_list);
+ gdev_unlock_restore(&vas->lock, &flags);
+ break;
+ case GDEV_MEM_DMA:
+ gdev_lock_save(&vas->lock, &flags);
+ gdev_list_add(&mem->list_entry_heap, &vas->dma_mem_list);
+ gdev_unlock_restore(&vas->lock, &flags);
+ break;
+ default:
+ GDEV_PRINT("Memory type not supported\n");
+ }
+}
+
+/* delete the memory object from the memory list. */
+static void __gdev_mem_list_del(struct gdev_mem *mem)
+{
+ struct gdev_vas *vas = mem->vas;
+ unsigned long flags;
+ int type = mem->type;
+
+ switch (type) {
+ case GDEV_MEM_DEVICE:
+ gdev_lock_save(&vas->lock, &flags);
+ gdev_list_del(&mem->list_entry_heap);
+ gdev_unlock_restore(&vas->lock, &flags);
+ break;
+ case GDEV_MEM_DMA:
+ gdev_lock_save(&vas->lock, &flags);
+ gdev_list_del(&mem->list_entry_heap);
+ gdev_unlock_restore(&vas->lock, &flags);
+ break;
+ default:
+ GDEV_PRINT("Memory type not supported\n");
+ }
+}
+
+/* allocate a new memory object. */
+struct gdev_mem *gdev_mem_alloc(struct gdev_vas *vas, uint64_t size, int type)
+{
+ struct gdev_mem *mem;
+ uint64_t addr;
+ void *map;
+
+ switch (type) {
+ case GDEV_MEM_DEVICE:
+ if (!(mem = gdev_raw_mem_alloc(vas, &addr, &size, &map)))
+ goto fail;
+ break;
+ case GDEV_MEM_DMA:
+ if (!(mem = gdev_raw_mem_alloc_dma(vas, &addr, &size, &map)))
+ goto fail;
+ break;
+ default:
+ GDEV_PRINT("Memory type not supported\n");
+ goto fail;
+ }
+
+ __gdev_mem_init(mem, vas, addr, size, map, type);
+ __gdev_mem_list_add(mem, type);
+
+ return mem;
+
+fail:
+ return NULL;
+}
+
+/* share memory space with @mem. if @mem is null, find a victim instead. */
+struct gdev_mem *gdev_mem_share(struct gdev_vas *vas, uint64_t size)
+{
+ struct gdev_device *gdev = vas->gdev;
+ struct gdev_mem *new;
+
+ /* request share memory with any memory object. */
+ gdev_mutex_lock(&gdev->shm_mutex);
+ if (!(new = gdev_shm_attach(vas, NULL, size)))
+ goto fail;
+ gdev_mutex_unlock(&gdev->shm_mutex);
+
+ __gdev_mem_list_add(new, new->type);
+
+ return new;
+
+fail:
+ gdev_mutex_unlock(&gdev->shm_mutex);
+ return NULL;
+}
+
+/* free the specified memory object. */
+void gdev_mem_free(struct gdev_mem *mem)
+{
+ struct gdev_vas *vas = mem->vas;
+ struct gdev_device *gdev = vas->gdev;
+
+ __gdev_mem_list_del(mem);
+
+ /* if the memory object is associated with shared memory, detach the
+ shared memory. note that the memory object will be freed if users
+ become zero.
+ free the memroy object otherwise. */
+ gdev_mutex_lock(&gdev->shm_mutex);
+ if (mem->shm)
+ gdev_shm_detach(mem);
+ else
+ gdev_raw_mem_free(mem);
+ gdev_mutex_unlock(&gdev->shm_mutex);
+}
+
+/* garbage collection: free all memory left in heap. */
+void gdev_mem_gc(struct gdev_vas *vas)
+{
+ struct gdev_mem *mem;
+
+ /* device memory. */
+ gdev_list_for_each (mem, &vas->mem_list, list_entry_heap) {
+ gdev_mem_free(mem);
+ }
+
+ /* host DMA memory. */
+ gdev_list_for_each (mem, &vas->dma_mem_list, list_entry_heap) {
+ gdev_mem_free(mem);
+ }
+}
+
+/* look up the memory object allocated at the specified address. */
+struct gdev_mem *gdev_mem_lookup(struct gdev_vas *vas, uint64_t addr, int type)
+{
+ struct gdev_mem *mem = NULL;
+ unsigned long flags;
+
+ switch (type) {
+ case GDEV_MEM_DEVICE:
+ gdev_lock_save(&vas->lock, &flags);
+ gdev_list_for_each (mem, &vas->mem_list, list_entry_heap) {
+ if (mem && (addr >= mem->addr && addr < mem->addr + mem->size))
+ break;
+ }
+ gdev_unlock_restore(&vas->lock, &flags);
+ break;
+ case GDEV_MEM_DMA:
+ gdev_lock_save(&vas->lock, &flags);
+ gdev_list_for_each (mem, &vas->dma_mem_list, list_entry_heap) {
+ uint64_t map_addr = (uint64_t) mem->map;
+ if (mem && (addr >= map_addr && addr < map_addr + mem->size))
+ break;
+ }
+ gdev_unlock_restore(&vas->lock, &flags);
+ break;
+ default:
+ GDEV_PRINT("Memory type not supported\n");
+ }
+
+ return mem;
+}
+
+/* get host DMA buffer. */
+void *gdev_mem_get_buf(struct gdev_mem *mem)
+{
+ if (mem->type != GDEV_MEM_DMA)
+ return NULL;
+ return mem->map;
+}
+
+/* get virtual memory address. */
+uint64_t gdev_mem_get_addr(struct gdev_mem *mem)
+{
+ return mem->addr;
+}
+
+/* get allocated memory size. */
+uint64_t gdev_mem_get_size(struct gdev_mem *mem)
+{
+ return mem->size;
+}
+
View
521 common/gdev_nvidia_shm.c
@@ -0,0 +1,521 @@
+/*
+ * Copyright 2011 Shinpei Kato
+ *
+ * University of California, Santa Cruz
+ * Systems Research Lab.
+ *
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "gdev_device.h"
+
+#define GDEV_SHM_SEGMENT_COUNT 64 /* hardcoded */
+static struct gdev_mem *gdev_shm_owners[GDEV_SHM_SEGMENT_COUNT] = {
+ [0 ... GDEV_SHM_SEGMENT_COUNT - 1] = NULL
+};
+
+/**
+ * these functions must be used when evicting/reloading data.
+ * better implementations wanted!
+ */
+int gdev_callback_evict_to_host(void*, void*, uint64_t, uint64_t);
+int gdev_callback_evict_to_device(void*, uint64_t, uint64_t, uint64_t);
+int gdev_callback_reload_from_host(void*, uint64_t, void*, uint64_t);
+int gdev_callback_reload_from_device(void*, uint64_t, uint64_t, uint64_t);
+
+/* attach device memory and allocate host buffer for swap */
+static int __gdev_swap_attach(struct gdev_mem *mem)
+{
+ struct gdev_vas *vas = mem->vas;
+ struct gdev_device *gdev = vas->gdev;
+ struct gdev_mem *swap_mem;
+ uint64_t addr, size;
+ void *map, *swap_buf;
+
+ /* host buffer for swap. */
+ swap_buf = MALLOC(mem->size);
+ if (!swap_buf)
+ goto fail_swap_buf;
+ /* device memory for temporal swap (shared by others). */
+ if (GDEV_SWAP_MEM_SIZE > 0) {
+ swap_mem = gdev_raw_mem_share(vas, gdev->swap, &addr, &size, &map);
+ if (!swap_mem)
+ goto fail_swap_mem;
+ __gdev_mem_init(swap_mem, vas, addr, size, map, GDEV_MEM_DEVICE);
+ }
+ else {
+ swap_mem = NULL;
+ }
+ mem->swap_buf = swap_buf;
+ mem->swap_mem = swap_mem;
+
+ return 0;
+
+fail_swap_mem:
+ FREE(swap_buf);
+fail_swap_buf:
+ return -ENOMEM;
+}
+
+/* detach device memory and free host buffer for swap */
+static void __gdev_swap_detach(struct gdev_mem *mem)
+{
+ if (GDEV_SWAP_MEM_SIZE > 0)
+ gdev_raw_mem_unshare(mem->swap_mem);
+ FREE(mem->swap_buf);
+}
+
+void __gdev_shm_init(struct gdev_mem *mem, struct gdev_shm *shm)
+{
+ gdev_list_init(&shm->mem_list, NULL);
+ gdev_list_init(&shm->list_entry, (void*)shm);
+ gdev_mutex_init(&shm->mutex);
+ shm->prio = GDEV_PRIO_MIN;
+ shm->users = 1; /* count itself. */
+ shm->holder = NULL;
+ shm->bo = mem->bo;
+ shm->size = mem->size;
+ shm->key = 0;
+ shm->id = -1;
+ gdev_mutex_lock(&shm->mutex);
+ /* the victim object itself must be inserted into the list. */
+ gdev_list_add(&mem->list_entry_shm, &shm->mem_list);
+ mem->shm = shm;
+ gdev_mutex_unlock(&shm->mutex);
+}
+
+int gdev_shm_create(struct gdev_device *gdev, struct gdev_vas *vas, int key, uint64_t size, int flags)
+{
+ struct gdev_mem *mem;
+ struct gdev_shm *shm;
+ int id = -1;
+ int i;
+
+ /* key = 0 is not allowed to be used by users. */
+ if (key == 0)
+ return -1;
+
+ gdev_mutex_lock(&gdev->shm_mutex);
+
+ gdev_list_for_each(shm, &gdev->shm_list, list_entry) {
+ if (key == shm->key) {
+ id = shm->id;
+ break;
+ }
+ }
+
+ if (id < 0) {
+ if (!(mem = gdev_mem_alloc(vas, size, GDEV_MEM_DEVICE)))
+ goto fail_mem_alloc;
+ /* register the new shared memory segment. */
+ for (i = 0; i < GDEV_SHM_SEGMENT_COUNT; i++) {
+ if (!gdev_shm_owners[i]) {
+ gdev_shm_owners[i] = mem;
+ id = i;
+ if (!(shm = MALLOC(sizeof(*shm))))
+ goto fail_shm_malloc;
+ __gdev_shm_init(mem, shm);
+ shm->key = key;
+ shm->id = id;
+ break;
+ }
+ }
+ }
+
+ gdev_mutex_unlock(&gdev->shm_mutex);
+
+ return id;
+
+fail_shm_malloc:
+ gdev_mem_free(mem);
+fail_mem_alloc:
+ gdev_mutex_unlock(&gdev->shm_mutex);
+ return -1;
+}
+
+/**
+ * destroy the shared memory segment.
+ * FIXME: there is a security issue that anyone can destroy it...
+ */
+int gdev_shm_destroy(struct gdev_device *gdev, int id)
+{
+ struct gdev_mem *mem;
+ struct gdev_shm *shm;
+
+ if (id < 0 || id >= GDEV_SHM_SEGMENT_COUNT)
+ return -EINVAL;
+
+ gdev_mutex_lock(&gdev->shm_mutex);
+ mem = gdev_shm_owners[id];
+ shm = mem->shm;
+ gdev_list_del(&mem->list_entry_shm);
+ gdev_list_del(&shm->list_entry);
+ FREE(shm);
+ gdev_mem_free(mem);
+ gdev_shm_owners[id] = NULL;
+ gdev_mutex_unlock(&gdev->shm_mutex);
+
+ return 0;
+}
+
+/* find a memory object that we can borrow some memory space from. */
+static struct gdev_mem *__gdev_shm_find_victim(struct gdev_vas *vas, uint64_t size)
+{
+ struct gdev_device *gdev = vas->gdev;
+ struct gdev_vas *v;
+ struct gdev_mem *m, *victim = NULL;
+ struct gdev_shm *shm;
+ unsigned long flags;
+
+ if (!(shm = MALLOC(sizeof(*shm))))
+ goto fail_shm;
+
+ /* select the lowest-priority object. */
+ gdev_lock_save(&gdev->vas_lock, &flags);
+ gdev_list_for_each (v, &gdev->vas_list, list_entry) {
+ gdev_lock_nested(&v->lock);
+ gdev_list_for_each (m, &v->mem_list, list_entry_heap) {
+ /* don't select from the save VAS object! */
+ if (m->size >= size && m->vas != vas) {
+ if (!victim)
+ victim = m;
+ else {
+ int victim_prio, m_prio;
+ if (victim->shm)
+ victim_prio = victim->shm->prio;
+ else
+ victim_prio = victim->vas->prio;
+ if (m->shm)
+ m_prio = m->shm->prio;
+ else
+ m_prio = m->vas->prio;
+ if (victim_prio > m_prio)
+ victim = m;
+ }
+ }
+ }
+ gdev_unlock_nested(&v->lock);
+ }
+ gdev_unlock_restore(&gdev->vas_lock, &flags);
+
+ if (!victim) {
+ FREE(shm);
+ }
+ /* if the victim object doesn't have a shared memory object yet,
+ allocate a new one here. note that this shared memory doesn't need
+ a key search, and hence is not inserted into the shared memory list. */
+ else if (!victim->shm) {
+ /* attach swap for evicting data from shared memory space. */
+ if (__gdev_swap_attach(victim))
+ goto fail_swap;
+ __gdev_shm_init(victim, shm);
+ }
+ else {
+ FREE(shm); /* shm was unused. */
+ }
+
+ return victim;
+
+fail_swap:
+ FREE(shm);
+fail_shm:
+ return NULL;
+}
+
+/* share memory space with @mem. if @mem is null, find victim instead. */
+struct gdev_mem *gdev_shm_attach(struct gdev_vas *vas, struct gdev_mem *mem, uint64_t size)
+{
+ struct gdev_mem *new;
+ uint64_t addr;
+ void *map;
+
+ if (!mem) {
+ /* select a victim memory object. victim->shm will be newly
+ allocated if NULL, with shm->users being incremented. */
+ if (!(mem = __gdev_shm_find_victim(vas, size)))
+ goto fail_victim;
+ }
+
+ /* borrow the same (physical) memory space by sharing. */
+ if (!(new = gdev_raw_mem_share(vas, mem, &addr, &size, &map)))
+ goto fail_shm;
+
+ /* initialize the new memory object. */
+ __gdev_mem_init(new, vas, addr, size, map, GDEV_MEM_DEVICE);
+
+ /* attach swap for evicting data from shared memory space. */
+ if (__gdev_swap_attach(new))
+ goto fail_swap;
+
+ gdev_mutex_lock(&mem->shm->mutex);
+ new->shm = mem->shm;
+ /* set the highest prio among users to the shared memory's priority. */
+ if (new->shm->prio < vas->prio)
+ new->shm->prio = vas->prio;
+ /* insert the new memory object into the shared memory list. */
+ gdev_list_add(&new->list_entry_shm, &new->shm->mem_list);
+ gdev_mutex_unlock(&mem->shm->mutex);
+
+ /* this increment is protected by gdev->shm_mutex. */
+ mem->shm->users++;
+
+ return new;
+
+fail_swap:
+ gdev_raw_mem_unshare(new);
+fail_shm:
+ mem->shm->users--;
+fail_victim:
+
+ return NULL;
+}
+
+void gdev_shm_detach(struct gdev_mem *mem)
+{
+ struct gdev_vas *vas = mem->vas;
+ struct gdev_shm *shm = mem->shm;
+ struct gdev_mem *m;
+
+ /* this decrement is protected by gdev->shm_mutex. */
+ shm->users--;
+
+ /* enter the local shared memory lock. */
+ gdev_mutex_lock(&shm->mutex);
+ mem->shm = NULL;
+ gdev_list_del(&mem->list_entry_shm);
+ __gdev_swap_detach(mem);
+ /* if the memory object is shared but no users, free it.
+ since users == 0, no one else will use mem->shm. */
+ if (shm->users == 0) {
+ /* freeing memory must be exclusive with using shared memory. */
+ gdev_raw_mem_free(mem);
+ gdev_mutex_unlock(&shm->mutex);
+ FREE(shm);
+ }
+ /* otherwise, just unshare the memory object. */
+ else {
+ /* if a freeing memory object has the highest priority among the
+ shared memory objects, find the next highest priority one. */
+ if (shm->prio == vas->prio) {
+ int prio = GDEV_PRIO_MIN;
+ gdev_list_for_each (m, &shm->mem_list, list_entry_shm) {
+ if (m->vas->prio > prio)
+ prio = m->vas->prio;
+ }
+ shm->prio = prio;
+ }
+ if (shm->holder == mem)
+ shm->holder = NULL;
+ /* unsharing memory must be exclusive with using shared memory. */
+ gdev_raw_mem_unshare(mem);
+ gdev_mutex_unlock(&shm->mutex);
+ }
+}
+
+/* evict the shared memory object data.
+ the shared memory object associated with @mem must be locked. */
+int gdev_shm_evict(struct gdev_ctx *ctx, struct gdev_mem *mem)
+{
+ struct gdev_vas *vas = mem->vas;
+ struct gdev_device *gdev = vas->gdev;
+ struct gdev_mem *holder;
+ uint64_t src_addr;
+ uint64_t size;
+ int ret;
+
+ if (mem->shm) {
+ void *h = ctx->vas->handle;
+ if (mem->shm->holder && mem->shm->holder != mem) {
+ struct gdev_mem *dev_swap = gdev->swap;
+ holder = mem->shm->holder;
+ src_addr = mem->addr;
+ size = mem->shm->size;
+ if (dev_swap && !dev_swap->shm->holder) {
+ uint64_t dst_addr = holder->swap_mem->addr;
+ ret = gdev_callback_evict_to_device(h,dst_addr,src_addr,size);
+ if (ret)
+ goto fail_evict;
+ dev_swap->shm->holder = mem;
+ }
+ else {
+ void *dst_buf = holder->swap_buf;
+ ret = gdev_callback_evict_to_host(h,dst_buf,src_addr,size);
+ if (ret)
+ goto fail_evict;
+ }
+ holder->evicted = 1;
+ }
+ mem->shm->holder = mem;
+ }
+
+ return 0;
+
+fail_evict:
+ return ret;
+}
+
+/* evict all the shared memory object data associated to @vas.
+ all the shared memory objects associated to @vas must be locked. */
+int gdev_shm_evict_all(struct gdev_ctx *ctx, struct gdev_vas *vas)
+{
+ struct gdev_mem *mem;
+
+ gdev_list_for_each (mem, &vas->mem_list, list_entry_heap) {
+ gdev_shm_evict(ctx, mem);
+ }
+
+ return 0;
+}
+
+/* reload the evicted memory object data.
+ the shared memory object associated with @mem must be locked. */
+int gdev_shm_reload(struct gdev_ctx *ctx, struct gdev_mem *mem)
+{
+ struct gdev_vas *vas = ctx->vas;
+ struct gdev_device *gdev = vas->gdev;
+ struct gdev_mem *dev_swap;
+ uint64_t dst_addr;
+ uint64_t size;
+ int ret;
+
+ if (mem->evicted) {
+ void *h = vas->handle;
+ /* evict the corresponding memory space first. */
+ gdev_shm_evict(ctx, mem);
+ /* reload data regardless whether eviction succeeded or failed. */
+ dev_swap = gdev->swap;
+ dst_addr = mem->addr;
+ size = mem->size;
+ if (dev_swap && dev_swap->shm->holder == mem) {
+ uint64_t src_addr = mem->swap_mem->addr;
+ ret = gdev_callback_reload_from_device(h, dst_addr, src_addr, size);
+ if (ret)
+ goto fail_reload;
+ }
+ else {
+ void *src_buf = mem->swap_buf;
+ ret = gdev_callback_reload_from_host(h, dst_addr, src_buf, size);
+ if (ret)
+ goto fail_reload;
+ }
+ mem->evicted = 0;
+ mem->shm->holder = mem;
+ }
+
+ return 0;
+
+fail_reload:
+ return ret;
+}
+
+/* reload all the evicted memory object data associated to @vas.
+ all the shared memory objects associated to @vas must be locked. */
+int gdev_shm_reload_all(struct gdev_ctx *ctx, struct gdev_vas *vas)
+{
+ struct gdev_mem *mem;
+
+ gdev_list_for_each (mem, &vas->mem_list, list_entry_heap) {
+ gdev_shm_reload(ctx, mem);
+ }
+
+ return 0;
+}
+
+/* lock the shared memory associated with @mem, if any. */
+void gdev_shm_lock(struct gdev_mem *mem)
+{
+ if (mem->shm) {
+ gdev_mutex_lock(&mem->shm->mutex);
+ }
+}
+
+/* unlock the shared memory associated with @mem, if any. */
+void gdev_shm_unlock(struct gdev_mem *mem)
+{
+ if (mem->shm) {
+ gdev_mutex_unlock(&mem->shm->mutex);
+ }
+}
+
+/* lock all the shared memory objects associated with @vas. */
+void gdev_shm_lock_all(struct gdev_vas *vas)
+{
+ struct gdev_device *gdev = vas->gdev;
+ struct gdev_mem *mem;
+
+ gdev_mutex_lock(&gdev->shm_mutex);
+ gdev_list_for_each (mem, &vas->mem_list, list_entry_heap) {
+ gdev_shm_lock(mem);
+ }
+ gdev_mutex_unlock(&gdev->shm_mutex);
+}
+
+/* unlock all the shared memory objects associated with @vas. */
+void gdev_shm_unlock_all(struct gdev_vas *vas)
+{
+ struct gdev_device *gdev = vas->gdev;
+ struct gdev_mem *mem;
+
+ gdev_mutex_lock(&gdev->shm_mutex);
+ gdev_list_for_each (mem, &vas->mem_list, list_entry_heap) {
+ gdev_shm_unlock(mem);
+ }
+ gdev_mutex_unlock(&gdev->shm_mutex);
+}
+
+/* create swap memory object for the device. */
+int gdev_swap_create(struct gdev_device *gdev, uint32_t size)
+{
+ struct gdev_mem *swap;
+ struct gdev_shm *shm;
+ int ret = 0;
+
+ swap = gdev_raw_swap_alloc(gdev, size);
+ if (swap) {
+ shm = MALLOC(sizeof(*shm));
+ if (shm)
+ shm->holder = NULL;
+ else {
+ gdev_raw_swap_free(swap);
+ swap = NULL;
+ ret = -ENOMEM;
+ }
+ swap->shm = shm;
+ }
+ else {
+ ret = -ENOMEM;
+ }
+
+ gdev->swap = swap;
+
+ return ret;
+}
+
+/* remove swap memory object from the device. */
+void gdev_swap_destroy(struct gdev_device *gdev)
+{
+ if (gdev->swap) {
+ if (gdev->swap->shm)
+ FREE(gdev->swap->shm);
+ gdev_raw_swap_free(gdev->swap);
+ }
+}
View
2 driver/pscnv/Makefile
@@ -19,7 +19,7 @@ pscnv-y := nouveau_drv.o nouveau_state.o \
pscnv_gdev.o \
gdev_drv.o gdev_fops.o gdev_ioctl.o gdev_proc.o \
gdev_api.o gdev_device.o gdev_sched.o \
- gdev_nvidia.o gdev_nvidia_nvc0.o
+ gdev_nvidia.o gdev_nvidia_compute.o gdev_nvidia_mem.o gdev_nvidia_shm.o gdev_nvidia_nvc0.o
pscnv-$(CONFIG_DRM_NOUVEAU_DEBUG) += nouveau_debugfs.o
pscnv-$(CONFIG_COMPAT) += nouveau_ioc32.o
View
15 runtime/user/gdev/Makefile
@@ -1,16 +1,15 @@
# Makefile
-CC = gcc
-TARGET = libgdev
-CFLAGS = -O2 -Wall
+CC = gcc
+TARGET = libgdev
+CFLAGS = -O2 -Wall
GDEVDIR = /usr/local/gdev
HEADERS = gdev_api.h,gdev_nvidia_def.h,gdev_list.h,gdev_time.h
-#OBJS = $(patsubst %.c,%.o,$(wildcard ./*.c))
-OBJS = pscnv_gdev.o libpscnv.o libpscnv_ib.o \
- gdev_lib.o \
- gdev_api.o gdev_device.o gdev_sched.o \
- gdev_nvidia.o gdev_nvidia_nvc0.o
+OBJS = pscnv_gdev.o libpscnv.o libpscnv_ib.o \
+ gdev_lib.o \
+ gdev_api.o gdev_device.o gdev_sched.o \
+ gdev_nvidia.o gdev_nvidia_compute.o gdev_nvidia_mem.o gdev_nvidia_shm.o gdev_nvidia_nvc0.o
ZOMBIE = $(wildcard ./*~)

0 comments on commit 6b4ff38

Please sign in to comment.