Permalink
Browse files

gdev: added more functions for shared memory support

  • Loading branch information...
1 parent 6b4ff38 commit 6584e350d0ec57fea00f8813801e8916ab494c9d Shinpei Kato committed Jan 7, 2012
Showing with 260 additions and 166 deletions.
  1. +99 −24 common/gdev_api.c
  2. +9 −1 common/gdev_api.h
  3. +10 −10 common/gdev_arch.h
  4. +1 −0 common/gdev_nvidia.h
  5. +1 −1 common/gdev_nvidia_compute.c
  6. +44 −0 common/gdev_nvidia_mem.c
  7. +96 −130 common/gdev_nvidia_shm.c
View
@@ -275,12 +275,12 @@ static int __gmemcpy_to_device
if (!mem)
return -ENOENT;
- gdev_shm_lock(mem);
- gdev_shm_evict(ctx, mem); /* evict data only if necessary */
+ gdev_mem_lock(mem);
+ gdev_shm_evict_conflict(ctx, mem); /* evict conflicting data. */
ret = __gmemcpy_to_device_locked(ctx, dst_addr, src_buf, size, async,
ch_size, p_count, vas, mem, dma_mem,
host_copy);
- gdev_shm_unlock(mem);
+ gdev_mem_unlock(mem);
return ret;
}
@@ -463,20 +463,20 @@ static int __gmemcpy_from_device
if (!mem)
return -ENOENT;
- gdev_shm_lock(mem);
- gdev_shm_reload(ctx, mem); /* reload data only if necessary. */
+ gdev_mem_lock(mem);
+ gdev_shm_retrieve_swap(ctx, mem); /* retrieve data swapped. */
ret = __gmemcpy_from_device_locked(ctx, dst_buf, src_addr, size, async,
ch_size, p_count, vas, mem, dma_mem,
host_copy);
- gdev_shm_unlock(mem);
+ gdev_mem_unlock(mem);
return ret;
}
/**
- * this function must be used when evicting data to host.
+ * this function must be used when saving data to host.
*/
-int gdev_callback_evict_to_host
+int gdev_callback_save_to_host
(void *h, void* dst_buf, uint64_t src_addr, uint64_t size)
{
gdev_vas_t *vas = ((struct gdev_handle*)h)->vas;
@@ -492,9 +492,9 @@ int gdev_callback_evict_to_host
}
/**
- * this function must be used when evicting data to device.
+ * this function must be used when saving data to device.
*/
-int gdev_callback_evict_to_device
+int gdev_callback_save_to_device
(void *h, uint64_t dst_addr, uint64_t src_addr, uint64_t size)
{
gdev_ctx_t *ctx = ((struct gdev_handle*)h)->ctx;
@@ -507,9 +507,9 @@ int gdev_callback_evict_to_device
}
/**
- * this function must be used when reloading data from host.
+ * this function must be used when loading data from host.
*/
-int gdev_callback_reload_from_host
+int gdev_callback_load_from_host
(void *h, uint64_t dst_addr, void *src_buf, uint64_t size)
{
gdev_vas_t *vas = ((struct gdev_handle*)h)->vas;
@@ -525,9 +525,9 @@ int gdev_callback_reload_from_host
}
/**
- * this function must be used when reloading data from device.
+ * this function must be used when loading data from device.
*/
-int gdev_callback_reload_from_device
+int gdev_callback_load_from_device
(void *h, uint64_t dst_addr, uint64_t src_addr, uint64_t size)
{
gdev_ctx_t *ctx = ((struct gdev_handle*)h)->ctx;
@@ -861,12 +861,12 @@ int gmemcpy_in_device
if (!dst || !src)
return -ENOENT;
- gdev_shm_lock(dst);
- gdev_shm_lock(src);
+ gdev_mem_lock(dst);
+ gdev_mem_lock(src);
fence = gdev_memcpy(ctx, dst_addr, src_addr, size, 0);
gdev_poll(ctx, fence, NULL);
- gdev_shm_unlock(src);
- gdev_shm_unlock(dst);
+ gdev_mem_unlock(src);
+ gdev_mem_unlock(dst);
return 0;
}
@@ -883,10 +883,10 @@ int glaunch(struct gdev_handle *h, struct gdev_kernel *kernel, uint32_t *id)
gdev_schedule_launch(se);
- gdev_shm_lock_all(vas);
- gdev_shm_reload_all(ctx, vas); /* this reloads data only if necessary */
+ gdev_mem_lock_all(vas);
+ gdev_shm_retrieve_swap_all(ctx, vas); /* get all data swapped back! */
*id = gdev_launch(ctx, kernel);
- gdev_shm_unlock_all(vas);
+ gdev_mem_unlock_all(vas);
return 0;
}
@@ -959,16 +959,91 @@ int gshmget(Ghandle h, int key, uint64_t size, int flags)
{
struct gdev_device *gdev = h->gdev;
gdev_vas_t *vas = h->vas;
-
- return gdev_shm_create(gdev, vas, key, size, flags);
+ int id;
+
+ gdev_mutex_lock(&gdev->shm_mutex);
+ id = gdev_shm_create(gdev, vas, key, size, flags);
+ gdev_mutex_unlock(&gdev->shm_mutex);
+
+ return id;
}
+/**
+ * gshmat():
+ * attach device shared memory.
+ * note that @addr and @flags are currently not supported.
+ */
uint64_t gshmat(Ghandle h, int id, uint64_t addr, int flags)
{
+ struct gdev_device *gdev = h->gdev;
+ gdev_vas_t *vas = h->vas;
+ gdev_mem_t *new, *owner;
+
+ gdev_mutex_lock(&gdev->shm_mutex);
+ if (!(owner = gdev_shm_lookup(gdev, id)))
+ goto fail;
+ if (!(new = gdev_shm_attach(vas, owner, gdev_mem_get_size(owner))))
+ goto fail;
+ gdev_mutex_unlock(&gdev->shm_mutex);
+
+ return gdev_mem_get_addr(new);
+
+fail:
+ gdev_mutex_unlock(&gdev->shm_mutex);
return 0;
}
-uint64_t gshmdt(Ghandle h, uint64_t addr)
+/**
+ * gshmdt():
+ * detach device shared memory.
+ */
+int gshmdt(Ghandle h, uint64_t addr)
{
+ struct gdev_device *gdev = h->gdev;
+ gdev_vas_t *vas = h->vas;
+ gdev_mem_t *mem;
+
+ gdev_mutex_lock(&gdev->shm_mutex);
+ if (!(mem = gdev_mem_lookup(vas, addr, GDEV_MEM_DEVICE)))
+ goto fail;
+ gdev_shm_detach(mem);
+ gdev_mutex_unlock(&gdev->shm_mutex);
+
return 0;
+
+fail:
+ gdev_mutex_unlock(&gdev->shm_mutex);
+ return -ENOENT;
+}
+
+/**
+ * gshmctl():
+ * control device shared memory.
+ */
+int gshmctl(Ghandle h, int id, int cmd, void *buf)
+{
+ struct gdev_device *gdev = h->gdev;
+ gdev_mem_t *owner;
+ int ret;
+
+ switch (cmd) {
+ case GDEV_IPC_RMID:
+ gdev_mutex_lock(&gdev->shm_mutex);
+ if (!(owner = gdev_shm_lookup(gdev, id))) {
+ ret = -ENOENT;
+ goto fail;
+ }
+ gdev_shm_destroy_mark(gdev, owner);
+ gdev_mutex_unlock(&gdev->shm_mutex);
+ break;
+ default:
+ GDEV_PRINT("gshmctl(): cmd %d not supported\n", cmd);
+ return -EINVAL;
+ }
+
+ return 0;
+
+fail:
+ gdev_mutex_unlock(&gdev->shm_mutex);
+ return ret;
}
View
@@ -63,7 +63,8 @@ int gquery(Ghandle h, uint32_t type, uint64_t *result);
int gtune(Ghandle h, uint32_t type, uint32_t value);
int gshmget(Ghandle h, int key, uint64_t size, int flags);
uint64_t gshmat(Ghandle h, int id, uint64_t addr, int flags);
-uint64_t gshmdt(Ghandle h, uint64_t addr);
+int gshmdt(Ghandle h, uint64_t addr);
+int gshmctl(Ghandle h, int id, int cmd, void *buf);
/**
@@ -79,4 +80,11 @@ uint64_t gshmdt(Ghandle h, uint64_t addr);
#define GDEV_QUERY_DMA_MEM_SIZE 2
#define GDEV_QUERY_CHIPSET 3
+/**
+ * IPC commands:
+ */
+#define GDEV_IPC_STAT 1
+#define GDEV_IPC_SET 2
+#define GDEV_IPC_RMID 3
+
#endif
View
@@ -69,21 +69,21 @@ gdev_mem_t *gdev_mem_share(gdev_vas_t *vas, uint64_t size);
void gdev_mem_free(gdev_mem_t *mem);
void gdev_mem_gc(gdev_vas_t *vas);
gdev_mem_t *gdev_mem_lookup(gdev_vas_t *vas, uint64_t addr, int type);
+void gdev_mem_lock(gdev_mem_t *mem);
+void gdev_mem_unlock(gdev_mem_t *mem);
+void gdev_mem_lock_all(gdev_vas_t *vas);
+void gdev_mem_unlock_all(gdev_vas_t *vas);
void *gdev_mem_get_buf(gdev_mem_t *mem);
uint64_t gdev_mem_get_addr(gdev_mem_t *mem);
uint64_t gdev_mem_get_size(gdev_mem_t *mem);
int gdev_shm_create(struct gdev_device *gdev, gdev_vas_t *vas, int key, uint64_t size, int flags);
-int gdev_shm_destroy(struct gdev_device *gdev, int id);
+int gdev_shm_destroy_mark(struct gdev_device *gdev, gdev_mem_t *owner);
gdev_mem_t *gdev_shm_attach(gdev_vas_t *vas, gdev_mem_t *mem, uint64_t size);
-void gdev_shm_detach(struct gdev_mem *mem);
-int gdev_shm_evict(gdev_ctx_t *ctx, gdev_mem_t *mem);
-int gdev_shm_evict_all(gdev_ctx_t *ctx, gdev_vas_t *vas);
-int gdev_shm_reload(gdev_ctx_t *ctx, gdev_mem_t *mem);
-int gdev_shm_reload_all(gdev_ctx_t *ctx, gdev_vas_t *vas);
-void gdev_shm_lock(gdev_mem_t *mem);
-void gdev_shm_unlock(gdev_mem_t *mem);
-void gdev_shm_lock_all(gdev_vas_t *vas);
-void gdev_shm_unlock_all(gdev_vas_t *vas);
+void gdev_shm_detach(gdev_mem_t *mem);
+gdev_mem_t *gdev_shm_lookup(struct gdev_device *gdev, int id);
+int gdev_shm_evict_conflict(gdev_ctx_t *ctx, gdev_mem_t *mem);
+int gdev_shm_retrieve_swap(gdev_ctx_t *ctx, gdev_mem_t *mem);
+int gdev_shm_retrieve_swap_all(gdev_ctx_t *ctx, gdev_vas_t *vas);
int gdev_swap_create(struct gdev_device *gdev, uint32_t size);
void gdev_swap_destroy(struct gdev_device *gdev);
View
@@ -70,6 +70,7 @@ struct gdev_shm {
int users; /* number of users (effective only for master) */
int key; /* key value of this shared memory */
int id; /* indentifier of this shared memory */
+ int implicit; /* true if created due to a lack of memory space. */
void *bo; /* private buffer object */
};
@@ -82,7 +82,7 @@ uint32_t gdev_launch(struct gdev_ctx *ctx, struct gdev_kernel *kern)
/* evict data saved in device swap memory space to host memory. */
if (dev_swap && dev_swap->shm->holder) {
struct gdev_mem *mem = dev_swap->shm->holder;
- gdev_shm_evict(ctx, mem->swap_mem); /* don't pass gdev->swap! */
+ gdev_shm_evict_conflict(ctx, mem->swap_mem); /* don't use gdev->swap */
dev_swap->shm->holder = NULL;
}
View
@@ -191,6 +191,50 @@ struct gdev_mem *gdev_mem_lookup(struct gdev_vas *vas, uint64_t addr, int type)
return mem;
}
+/* lock the memory object so that none can change data while tranferring.
+ if there are no shared memory users, no need to lock. */
+void gdev_mem_lock(struct gdev_mem *mem)
+{
+ if (mem->shm) {
+ gdev_mutex_lock(&mem->shm->mutex);
+ }
+}
+
+/* unlock the memory object so that none can change data while tranferring.
+ if there are no shared memory users, no need to lock. */
+void gdev_mem_unlock(struct gdev_mem *mem)
+{
+ if (mem->shm) {
+ gdev_mutex_unlock(&mem->shm->mutex);
+ }
+}
+
+/* lock all the memory objects associated with @vas. */
+void gdev_mem_lock_all(struct gdev_vas *vas)
+{
+ struct gdev_device *gdev = vas->gdev;
+ struct gdev_mem *mem;
+
+ gdev_mutex_lock(&gdev->shm_mutex);
+ gdev_list_for_each (mem, &vas->mem_list, list_entry_heap) {
+ gdev_mem_lock(mem);
+ }
+ gdev_mutex_unlock(&gdev->shm_mutex);
+}
+
+/* unlock all the memory objects associated with @vas. */
+void gdev_mem_unlock_all(struct gdev_vas *vas)
+{
+ struct gdev_device *gdev = vas->gdev;
+ struct gdev_mem *mem;
+
+ gdev_mutex_lock(&gdev->shm_mutex);
+ gdev_list_for_each (mem, &vas->mem_list, list_entry_heap) {
+ gdev_mem_unlock(mem);
+ }
+ gdev_mutex_unlock(&gdev->shm_mutex);
+}
+
/* get host DMA buffer. */
void *gdev_mem_get_buf(struct gdev_mem *mem)
{
Oops, something went wrong.

0 comments on commit 6584e35

Please sign in to comment.