Permalink
Browse files

gdev: fixed global locking mechanim

  • Loading branch information...
1 parent c42144d commit a67ac73449dd63d3bee0833a67f35de0c90a2a4c Shinpei Kato committed Jan 28, 2012
Showing with 116 additions and 26 deletions.
  1. +27 −5 common/gdev_api.c
  2. +4 −2 common/gdev_arch.h
  3. +3 −0 common/gdev_device.c
  4. +3 −0 common/gdev_device.h
  5. +67 −19 common/gdev_nvidia.c
  6. +12 −0 common/gdev_sched.c
View
32 common/gdev_api.c
@@ -250,6 +250,7 @@ static int __gmemcpy_to_device_locked(gdev_ctx_t *ctx, uint64_t dst_addr, const
*/
static int __gmemcpy_to_device(struct gdev_handle *h, uint64_t dst_addr, const void *src_buf, uint64_t size, uint32_t *id, int (*host_copy)(void*, const void*, uint32_t))
{
+ struct gdev_device *gdev = h->gdev;
gdev_vas_t *vas = h->vas;
gdev_ctx_t *ctx = h->ctx;
gdev_mem_t *mem = gdev_mem_lookup(vas, dst_addr, GDEV_MEM_DEVICE);
@@ -265,16 +266,16 @@ static int __gmemcpy_to_device(struct gdev_handle *h, uint64_t dst_addr, const v
/* decide if the context needs to stall or not. */
gdev_schedule_memory(se);
- /* memcopy operation needs to be protected by the lock. */
gdev_mem_lock(mem);
+
gdev_shm_evict_conflict(ctx, mem); /* evict conflicting data. */
ret = __gmemcpy_to_device_locked(ctx, dst_addr, src_buf, size, id,
ch_size, p_count, vas, mem, dma_mem,
host_copy);
gdev_mem_unlock(mem);
/* select the next context by itself, since memcpy is sychronous. */
- gdev_select_next_memory(h->gdev);
+ gdev_select_next_memory(gdev);
return ret;
}
@@ -431,6 +432,7 @@ static int __gmemcpy_from_device_locked(gdev_ctx_t *ctx, void *dst_buf, uint64_t
*/
static int __gmemcpy_from_device(struct gdev_handle *h, void *dst_buf, uint64_t src_addr, uint64_t size, uint32_t *id, int (*host_copy)(void*, const void*, uint32_t))
{
+ struct gdev_device *gdev = h->gdev;
gdev_vas_t *vas = h->vas;
gdev_ctx_t *ctx = h->ctx;
gdev_mem_t *mem = gdev_mem_lookup(vas, src_addr, GDEV_MEM_DEVICE);
@@ -446,16 +448,17 @@ static int __gmemcpy_from_device(struct gdev_handle *h, void *dst_buf, uint64_t
/* decide if the context needs to stall or not. */
gdev_schedule_memory(se);
- /* memcpy operation needs to be protected by the lock. */
gdev_mem_lock(mem);
+
gdev_shm_retrieve_swap(ctx, mem); /* retrieve data swapped. */
ret = __gmemcpy_from_device_locked(ctx, dst_buf, src_addr, size, id,
ch_size, p_count, vas, mem, dma_mem,
host_copy);
+
gdev_mem_unlock(mem);
/* select the next context by itself, since memcpy is synchronous. */
- gdev_select_next_memory(h->gdev);
+ gdev_select_next_memory(gdev);
return ret;
}
@@ -552,6 +555,9 @@ struct gdev_handle *gopen(int minor)
goto fail_open;
}
+ /* none can access GPU while someone is opening device. */
+ gdev_block_start(gdev);
+
/* create a new virual address space (VAS) object. */
vas = gdev_vas_new(gdev, GDEV_VAS_SIZE, h);
if (!vas) {
@@ -580,6 +586,9 @@ struct gdev_handle *gopen(int minor)
goto fail_se;
}
+ /* now other users can access the GPU. */
+ gdev_block_end(gdev);
+
/* save the objects to the handle. */
h->se = se;
h->dma_mem = dma_mem;
@@ -599,6 +608,7 @@ struct gdev_handle *gopen(int minor)
fail_ctx:
gdev_vas_free(vas);
fail_vas:
+ gdev_block_end(gdev);
gdev_dev_close(gdev);
fail_open:
return NULL;
@@ -824,6 +834,8 @@ int gmemcpy_user_from_device_async(struct gdev_handle *h, void *dst_buf, uint64_
int gmemcpy_in_device
(struct gdev_handle *h, uint64_t dst_addr, uint64_t src_addr, uint64_t size)
{
+ struct gdev_device *gdev = h->gdev;
+ struct gdev_sched_entity *se = h->se;
gdev_ctx_t *ctx = h->ctx;
gdev_vas_t *vas = h->vas;
gdev_mem_t *dst = gdev_mem_lookup(vas, dst_addr, GDEV_MEM_DEVICE);
@@ -833,13 +845,21 @@ int gmemcpy_in_device
if (!dst || !src)
return -ENOENT;
+ /* decide if the context needs to stall or not. */
+ gdev_schedule_memory(se);
+
gdev_mem_lock(dst);
gdev_mem_lock(src);
+
fence = gdev_memcpy(ctx, dst_addr, src_addr, size);
gdev_poll(ctx, fence, NULL);
+
gdev_mem_unlock(src);
gdev_mem_unlock(dst);
+ /* select the next context by itself, since memcpy is synchronous. */
+ gdev_select_next_memory(gdev);
+
return 0;
}
@@ -849,16 +869,18 @@ int gmemcpy_in_device
*/
int glaunch(struct gdev_handle *h, struct gdev_kernel *kernel, uint32_t *id)
{
+ struct gdev_sched_entity *se = h->se;
gdev_vas_t *vas = h->vas;
gdev_ctx_t *ctx = h->ctx;
- struct gdev_sched_entity *se = h->se;
/* decide if the context needs to stall or not. */
gdev_schedule_compute(se);
gdev_mem_lock_all(vas);
+
gdev_shm_retrieve_swap_all(ctx, vas); /* get all data swapped back! */
*id = gdev_launch(ctx, kernel);
+
gdev_mem_unlock_all(vas);
return 0;
View
6 common/gdev_arch.h
@@ -65,8 +65,10 @@ void gdev_vas_free(gdev_vas_t *vas);
gdev_ctx_t *gdev_ctx_new(struct gdev_device *gdev, gdev_vas_t *vas);
void gdev_ctx_free(gdev_ctx_t *ctx);
int gdev_ctx_get_cid(gdev_ctx_t *ctx);
-void gdev_global_lock(struct gdev_device *gdev);
-void gdev_global_unlock(struct gdev_device *gdev);
+void gdev_block_start(struct gdev_device *gdev);
+void gdev_block_end(struct gdev_device *gdev);
+void gdev_access_start(struct gdev_device *gdev);
+void gdev_access_end(struct gdev_device *gdev);
void gdev_mem_lock(gdev_mem_t *mem);
void gdev_mem_unlock(gdev_mem_t *mem);
void gdev_mem_lock_all(gdev_vas_t *vas);
View
3 common/gdev_device.c
@@ -51,6 +51,8 @@ void __gdev_init_device(struct gdev_device *gdev, int id)
{
gdev->id = id;
gdev->users = 0;
+ gdev->accessed = 0;
+ gdev->blocked = 0;
gdev->mem_size = 0;
gdev->mem_used = 0;
gdev->dma_mem_size = 0;
@@ -80,6 +82,7 @@ void __gdev_init_device(struct gdev_device *gdev, int id)
gdev_lock_init(&gdev->sched_com_lock);
gdev_lock_init(&gdev->sched_mem_lock);
gdev_lock_init(&gdev->vas_lock);
+ gdev_lock_init(&gdev->global_lock);
gdev_mutex_init(&gdev->shm_mutex);
}
View
3 common/gdev_device.h
@@ -50,6 +50,8 @@
struct gdev_device {
int id; /* device ID */
int users; /* the number of threads/processes using the device */
+ int accessed; /* indicate if any process is on the device */
+ int blocked; /* incidate if a process is allowed to access the device */
uint32_t chipset;
uint64_t mem_size;
uint64_t mem_used;
@@ -81,6 +83,7 @@ struct gdev_device {
gdev_lock_t sched_com_lock;
gdev_lock_t sched_mem_lock;
gdev_lock_t vas_lock;
+ gdev_lock_t global_lock;
gdev_mutex_t shm_mutex;
gdev_mem_t *swap; /* reserved swap memory space */
};
View
86 common/gdev_nvidia.c
@@ -125,41 +125,89 @@ int gdev_ctx_get_cid(struct gdev_ctx *ctx)
return ctx->cid;
}
-/* lock the device globally. */
-void gdev_global_lock(struct gdev_device *gdev)
+/* set the flag to block any access to the device. */
+void gdev_block_start(struct gdev_device *gdev)
{
struct gdev_device *phys = gdev->parent;
-
+
+ /* we have to spin while some context is accessing the GPU. */
+retry:
if (phys) {
- int physid = phys->id;
- int i, j = 0;
- for (i = 0; i < physid; i++)
- j += VCOUNT_LIST[i];
- for (i = j; i < j + VCOUNT_LIST[physid]; i++) {
- gdev_mutex_lock(&gdev_vds[i].shm_mutex);
+ gdev_lock(&phys->global_lock);
+ if (phys->accessed) {
+ gdev_unlock(&phys->global_lock);
+ goto retry;
}
+ phys->blocked = 1;
+ gdev_unlock(&phys->global_lock);
}
else {
- gdev_mutex_lock(&gdev->shm_mutex);
+ gdev_lock(&gdev->global_lock);
+ if (gdev->accessed) {
+ gdev_unlock(&gdev->global_lock);
+ goto retry;
+ }
+ gdev->blocked = 1;
+ gdev_unlock(&gdev->global_lock);
}
}
-/* unlock the device globally. */
-void gdev_global_unlock(struct gdev_device *gdev)
+/* clear the flag to unlock any access to the device. */
+void gdev_block_end(struct gdev_device *gdev)
{
struct gdev_device *phys = gdev->parent;
if (phys) {
- int physid = phys->id;
- int i, j = 0;
- for (i = 0; i < physid; i++)
- j += VCOUNT_LIST[i];
- for (i = j + VCOUNT_LIST[physid] - 1; i >= j ; i--) {
- gdev_mutex_unlock(&gdev_vds[i].shm_mutex);
+ gdev_lock(&phys->global_lock);
+ phys->blocked = 0;
+ gdev_unlock(&phys->global_lock);
+ }
+ else {
+ gdev_lock(&gdev->global_lock);
+ gdev->blocked = 0;
+ gdev_unlock(&gdev->global_lock);
+ }
+}
+
+/* increment the counter for # of contexts accessing the device. */
+void gdev_access_start(struct gdev_device *gdev)
+{
+ struct gdev_device *phys = gdev->parent;
+
+retry:
+ if (phys) {
+ gdev_lock(&phys->global_lock);
+ if (phys->blocked) {
+ gdev_unlock(&phys->global_lock);
+ goto retry;
}
+ phys->accessed++;
+ gdev_unlock(&phys->global_lock);
}
else {
- gdev_mutex_unlock(&gdev->shm_mutex);
+ gdev_lock(&gdev->global_lock);
+ if (gdev->blocked) {
+ gdev_unlock(&gdev->global_lock);
+ goto retry;
+ }
+ gdev->accessed++;
+ gdev_unlock(&gdev->global_lock);
}
}
+/* decrement the counter for # of contexts accessing the device. */
+void gdev_access_end(struct gdev_device *gdev)
+{
+ struct gdev_device *phys = gdev->parent;
+
+ if (phys) {
+ gdev_lock(&phys->global_lock);
+ phys->accessed--;
+ gdev_unlock(&phys->global_lock);
+ }
+ else {
+ gdev_lock(&gdev->global_lock);
+ gdev->accessed--;
+ gdev_unlock(&gdev->global_lock);
+ }
+}
View
12 common/gdev_sched.c
@@ -51,18 +51,22 @@ void gdev_sched_entity_destroy(struct gdev_sched_entity *se)
}
void gdev_schedule_compute(struct gdev_sched_entity *se)
{
+ gdev_access_start(gdev);
}
void gdev_select_next_compute(struct gdev_device *gdev)
{
+ gdev_access_end(gdev);
}
void gdev_replenish_credit_compute(struct gdev_device *gdev)
{
}
void gdev_schedule_memory(struct gdev_sched_entity *se)
{
+ gdev_access_start(gdev);
}
void gdev_select_next_memory(struct gdev_device *gdev)
{
+ gdev_access_end(gdev);
}
void gdev_replenish_credit_memory(struct gdev_device *gdev)
{
@@ -258,6 +262,8 @@ void gdev_schedule_compute(struct gdev_sched_entity *se)
gdev->current_com = (void*)se;
gdev_unlock(&gdev->sched_com_lock);
}
+
+ gdev_access_start(gdev);
}
/**
@@ -270,6 +276,8 @@ void gdev_select_next_compute(struct gdev_device *gdev)
struct gdev_device *next;
struct gdev_time now, exec;
+ gdev_access_end(gdev);
+
gdev_lock(&gdev->sched_com_lock);
se = (struct gdev_sched_entity *)gdev->current_com;
if (!se) {
@@ -377,6 +385,8 @@ void gdev_schedule_memory(struct gdev_sched_entity *se)
gdev->current_mem = (void*)se;
gdev_unlock(&gdev->sched_mem_lock);
}
+
+ gdev_access_start(gdev);
}
/**
@@ -394,6 +404,8 @@ void gdev_select_next_memory(struct gdev_device *gdev)
return;
#endif
+ gdev_access_end(gdev);
+
gdev_lock(&gdev->sched_mem_lock);
se = (struct gdev_sched_entity *)gdev->current_mem;
if (!se) {

0 comments on commit a67ac73

Please sign in to comment.