gdev: fixed global locking mechanim

common/gdev_api.c

@@ -250,6 +250,7 @@ static int __gmemcpy_to_device_locked(gdev_ctx_t *ctx, uint64_t dst_addr, const
  */
 static int __gmemcpy_to_device(struct gdev_handle *h, uint64_t dst_addr, const void *src_buf, uint64_t size, uint32_t *id, int (*host_copy)(void*, const void*, uint32_t))
 {
+	struct gdev_device *gdev = h->gdev;
 	gdev_vas_t *vas = h->vas;
 	gdev_ctx_t *ctx = h->ctx;
 	gdev_mem_t *mem = gdev_mem_lookup(vas, dst_addr, GDEV_MEM_DEVICE);
@@ -265,16 +266,16 @@ static int __gmemcpy_to_device(struct gdev_handle *h, uint64_t dst_addr, const v
 	/* decide if the context needs to stall or not. */
 	gdev_schedule_memory(se);
 
-	/* memcopy operation needs to be protected by the lock. */
 	gdev_mem_lock(mem);
+
 	gdev_shm_evict_conflict(ctx, mem); /* evict conflicting data. */
 	ret = __gmemcpy_to_device_locked(ctx, dst_addr, src_buf, size, id, 
 									 ch_size, p_count, vas, mem, dma_mem, 
 									 host_copy);
 	gdev_mem_unlock(mem);
 
 	/* select the next context by itself, since memcpy is sychronous. */
-	gdev_select_next_memory(h->gdev);
+	gdev_select_next_memory(gdev);
 
 	return ret;
 }
@@ -431,6 +432,7 @@ static int __gmemcpy_from_device_locked(gdev_ctx_t *ctx, void *dst_buf, uint64_t
  */
 static int __gmemcpy_from_device(struct gdev_handle *h, void *dst_buf, uint64_t src_addr, uint64_t size, uint32_t *id, int (*host_copy)(void*, const void*, uint32_t))
 {
+	struct gdev_device *gdev = h->gdev;
 	gdev_vas_t *vas = h->vas;
 	gdev_ctx_t *ctx = h->ctx;
 	gdev_mem_t *mem = gdev_mem_lookup(vas, src_addr, GDEV_MEM_DEVICE);
@@ -446,16 +448,17 @@ static int __gmemcpy_from_device(struct gdev_handle *h, void *dst_buf, uint64_t
 	/* decide if the context needs to stall or not. */
 	gdev_schedule_memory(se);
 
-	/* memcpy operation needs to be protected by the lock. */
 	gdev_mem_lock(mem);
+
 	gdev_shm_retrieve_swap(ctx, mem); /* retrieve data swapped. */
 	ret = __gmemcpy_from_device_locked(ctx, dst_buf, src_addr, size, id, 
 									   ch_size, p_count, vas, mem, dma_mem,
 									   host_copy);
+
 	gdev_mem_unlock(mem);
 
 	/* select the next context by itself, since memcpy is synchronous. */
-	gdev_select_next_memory(h->gdev);
+	gdev_select_next_memory(gdev);
 
 	return ret;
 }
@@ -552,6 +555,9 @@ struct gdev_handle *gopen(int minor)
 		goto fail_open;
 	}
 
+	/* none can access GPU while someone is opening device. */
+	gdev_block_start(gdev);
+
 	/* create a new virual address space (VAS) object. */
 	vas = gdev_vas_new(gdev, GDEV_VAS_SIZE, h);
 	if (!vas) {
@@ -580,6 +586,9 @@ struct gdev_handle *gopen(int minor)
 		goto fail_se;
 	}
 
+	/* now other users can access the GPU. */
+	gdev_block_end(gdev);
+
 	/* save the objects to the handle. */
 	h->se = se;
 	h->dma_mem = dma_mem;
@@ -599,6 +608,7 @@ struct gdev_handle *gopen(int minor)
 fail_ctx:
 	gdev_vas_free(vas);
 fail_vas:
+	gdev_block_end(gdev);
 	gdev_dev_close(gdev);
 fail_open:
 	return NULL;
@@ -824,6 +834,8 @@ int gmemcpy_user_from_device_async(struct gdev_handle *h, void *dst_buf, uint64_
 int gmemcpy_in_device
 (struct gdev_handle *h, uint64_t dst_addr, uint64_t src_addr, uint64_t size)
 {
+	struct gdev_device *gdev = h->gdev;
+	struct gdev_sched_entity *se = h->se;
 	gdev_ctx_t *ctx = h->ctx;
 	gdev_vas_t *vas = h->vas;
 	gdev_mem_t *dst = gdev_mem_lookup(vas, dst_addr, GDEV_MEM_DEVICE);
@@ -833,13 +845,21 @@ int gmemcpy_in_device
 	if (!dst || !src)
 		return -ENOENT;
 
+	/* decide if the context needs to stall or not. */
+	gdev_schedule_memory(se);
+
 	gdev_mem_lock(dst);
 	gdev_mem_lock(src);
+
 	fence = gdev_memcpy(ctx, dst_addr, src_addr, size); 
 	gdev_poll(ctx, fence, NULL);
+
 	gdev_mem_unlock(src);
 	gdev_mem_unlock(dst);
 
+	/* select the next context by itself, since memcpy is synchronous. */
+	gdev_select_next_memory(gdev);
+
 	return 0;
 }
 
@@ -849,16 +869,18 @@ int gmemcpy_in_device
  */
 int glaunch(struct gdev_handle *h, struct gdev_kernel *kernel, uint32_t *id)
 {
+	struct gdev_sched_entity *se = h->se;
 	gdev_vas_t *vas = h->vas;
 	gdev_ctx_t *ctx = h->ctx;
-	struct gdev_sched_entity *se = h->se;
 
 	/* decide if the context needs to stall or not. */
 	gdev_schedule_compute(se);
 
 	gdev_mem_lock_all(vas);
+
 	gdev_shm_retrieve_swap_all(ctx, vas); /* get all data swapped back! */
 	*id = gdev_launch(ctx, kernel);
+
 	gdev_mem_unlock_all(vas);
 
 	return 0;

common/gdev_arch.h

@@ -65,8 +65,10 @@ void gdev_vas_free(gdev_vas_t *vas);
 gdev_ctx_t *gdev_ctx_new(struct gdev_device *gdev, gdev_vas_t *vas);
 void gdev_ctx_free(gdev_ctx_t *ctx);
 int gdev_ctx_get_cid(gdev_ctx_t *ctx);
-void gdev_global_lock(struct gdev_device *gdev);
-void gdev_global_unlock(struct gdev_device *gdev);
+void gdev_block_start(struct gdev_device *gdev);
+void gdev_block_end(struct gdev_device *gdev);
+void gdev_access_start(struct gdev_device *gdev);
+void gdev_access_end(struct gdev_device *gdev);
 void gdev_mem_lock(gdev_mem_t *mem);
 void gdev_mem_unlock(gdev_mem_t *mem);
 void gdev_mem_lock_all(gdev_vas_t *vas);

common/gdev_device.c

@@ -51,6 +51,8 @@ void __gdev_init_device(struct gdev_device *gdev, int id)
 {
 	gdev->id = id;
 	gdev->users = 0;
+	gdev->accessed = 0;
+	gdev->blocked = 0;
 	gdev->mem_size = 0;
 	gdev->mem_used = 0;
 	gdev->dma_mem_size = 0;
@@ -80,6 +82,7 @@ void __gdev_init_device(struct gdev_device *gdev, int id)
 	gdev_lock_init(&gdev->sched_com_lock);
 	gdev_lock_init(&gdev->sched_mem_lock);
 	gdev_lock_init(&gdev->vas_lock);
+	gdev_lock_init(&gdev->global_lock);
 	gdev_mutex_init(&gdev->shm_mutex);
 }
 

common/gdev_device.h

@@ -50,6 +50,8 @@
 struct gdev_device {
 	int id; /* device ID */
 	int users; /* the number of threads/processes using the device */
+	int accessed; /* indicate if any process is on the device */
+	int blocked; /* incidate if a process is allowed to access the device */
 	uint32_t chipset;
 	uint64_t mem_size;
 	uint64_t mem_used;
@@ -81,6 +83,7 @@ struct gdev_device {
 	gdev_lock_t sched_com_lock;
 	gdev_lock_t sched_mem_lock;
 	gdev_lock_t vas_lock;
+	gdev_lock_t global_lock;
 	gdev_mutex_t shm_mutex;
 	gdev_mem_t *swap; /* reserved swap memory space */
 };

common/gdev_nvidia.c

@@ -125,41 +125,89 @@ int gdev_ctx_get_cid(struct gdev_ctx *ctx)
 	return ctx->cid;
 }
 
-/* lock the device globally. */
-void gdev_global_lock(struct gdev_device *gdev)
+/* set the flag to block any access to the device. */
+void gdev_block_start(struct gdev_device *gdev)
 {
 	struct gdev_device *phys = gdev->parent;
-
+
+	/* we have to spin while some context is accessing the GPU. */
+retry:
 	if (phys) {
-		int physid = phys->id;
-		int i, j = 0;
-		for (i = 0; i < physid; i++)
-			j += VCOUNT_LIST[i];
-		for (i = j; i < j + VCOUNT_LIST[physid]; i++) {
-			gdev_mutex_lock(&gdev_vds[i].shm_mutex);
+		gdev_lock(&phys->global_lock);
+		if (phys->accessed) {
+			gdev_unlock(&phys->global_lock);
+			goto retry;
 		}
+		phys->blocked = 1;
+		gdev_unlock(&phys->global_lock);
 	}
 	else {
-		gdev_mutex_lock(&gdev->shm_mutex);
+		gdev_lock(&gdev->global_lock);
+		if (gdev->accessed) {
+			gdev_unlock(&gdev->global_lock);
+			goto retry;
+		}
+		gdev->blocked = 1;
+		gdev_unlock(&gdev->global_lock);
 	}
 }
 
-/* unlock the device globally. */
-void gdev_global_unlock(struct gdev_device *gdev)
+/* clear the flag to unlock any access to the device. */
+void gdev_block_end(struct gdev_device *gdev)
 {
 	struct gdev_device *phys = gdev->parent;
 
 	if (phys) {
-		int physid = phys->id;
-		int i, j = 0;
-		for (i = 0; i < physid; i++)
-			j += VCOUNT_LIST[i];
-		for (i = j + VCOUNT_LIST[physid] - 1; i >= j ; i--) {
-			gdev_mutex_unlock(&gdev_vds[i].shm_mutex);
+		gdev_lock(&phys->global_lock);
+		phys->blocked = 0;
+		gdev_unlock(&phys->global_lock);
+	}
+	else {
+		gdev_lock(&gdev->global_lock);
+		gdev->blocked = 0;
+		gdev_unlock(&gdev->global_lock);
+	}
+}
+
+/* increment the counter for # of contexts accessing the device. */
+void gdev_access_start(struct gdev_device *gdev)
+{
+	struct gdev_device *phys = gdev->parent;
+
+retry:
+	if (phys) {
+		gdev_lock(&phys->global_lock);
+		if (phys->blocked) {
+			gdev_unlock(&phys->global_lock);
+			goto retry;
 		}
+		phys->accessed++;
+		gdev_unlock(&phys->global_lock);
 	}
 	else {
-		gdev_mutex_unlock(&gdev->shm_mutex);
+		gdev_lock(&gdev->global_lock);
+		if (gdev->blocked) {
+			gdev_unlock(&gdev->global_lock);
+			goto retry;
+		}
+		gdev->accessed++;
+		gdev_unlock(&gdev->global_lock);
 	}
 }
 
+/* decrement the counter for # of contexts accessing the device. */
+void gdev_access_end(struct gdev_device *gdev)
+{
+	struct gdev_device *phys = gdev->parent;
+
+	if (phys) {
+		gdev_lock(&phys->global_lock);
+		phys->accessed--;
+		gdev_unlock(&phys->global_lock);
+	}
+	else {
+		gdev_lock(&gdev->global_lock);
+		gdev->accessed--;
+		gdev_unlock(&gdev->global_lock);
+	}
+}

common/gdev_sched.c

@@ -51,18 +51,22 @@ void gdev_sched_entity_destroy(struct gdev_sched_entity *se)
 }
 void gdev_schedule_compute(struct gdev_sched_entity *se)
 {
+	gdev_access_start(gdev);
 }
 void gdev_select_next_compute(struct gdev_device *gdev)
 {
+	gdev_access_end(gdev);
 }
 void gdev_replenish_credit_compute(struct gdev_device *gdev)
 {
 }
 void gdev_schedule_memory(struct gdev_sched_entity *se)
 {
+	gdev_access_start(gdev);
 }
 void gdev_select_next_memory(struct gdev_device *gdev)
 {
+	gdev_access_end(gdev);
 }
 void gdev_replenish_credit_memory(struct gdev_device *gdev)
 {
@@ -258,6 +262,8 @@ void gdev_schedule_compute(struct gdev_sched_entity *se)
 		gdev->current_com = (void*)se;
 		gdev_unlock(&gdev->sched_com_lock);
 	}
+
+	gdev_access_start(gdev);
 }
 
 /**
@@ -270,6 +276,8 @@ void gdev_select_next_compute(struct gdev_device *gdev)
 	struct gdev_device *next;
 	struct gdev_time now, exec;
 
+	gdev_access_end(gdev);
+
 	gdev_lock(&gdev->sched_com_lock);
 	se = (struct gdev_sched_entity *)gdev->current_com;
 	if (!se) {
@@ -377,6 +385,8 @@ void gdev_schedule_memory(struct gdev_sched_entity *se)
 		gdev->current_mem = (void*)se;
 		gdev_unlock(&gdev->sched_mem_lock);
 	}
+
+	gdev_access_start(gdev);
 }
 
 /**
@@ -394,6 +404,8 @@ void gdev_select_next_memory(struct gdev_device *gdev)
 	return;
 #endif
 
+	gdev_access_end(gdev);
+
 	gdev_lock(&gdev->sched_mem_lock);
 	se = (struct gdev_sched_entity *)gdev->current_mem;
 	if (!se) {