gdev: fixed the scheduler

gdev: fixed some bugs in the list functions

common/gdev_api.c

@@ -568,8 +568,10 @@ struct gdev_handle *gopen(int minor)
 	gdev_ctx_t *ctx;
 	gdev_mem_t **dma_mem;
 
-	if (!(h = MALLOC(sizeof(*h))))
+	if (!(h = MALLOC(sizeof(*h)))) {
+		GDEV_PRINT("Failed to allocate device handle\n");
 		return NULL;
+	}
 
 	h->pipeline_count = GDEV_PIPELINE_DEFAULT_COUNT;
 	h->chunk_size = GDEV_CHUNK_DEFAULT_SIZE;
@@ -617,7 +619,7 @@ struct gdev_handle *gopen(int minor)
 	h->gdev = gdev;
 	h->dev_id = minor;
 
-	GDEV_DPRINT("Opened gdev%d\n", minor);
+	GDEV_PRINT("Opened gdev%d\n", minor);
 
 	return h;
 
@@ -659,7 +661,7 @@ int gclose(struct gdev_handle *h)
 	gdev_vas_free(h->vas);
 	gdev_dev_close(h->gdev);
 
-	GDEV_DPRINT("Closed gdev%d\n", h->dev_id);
+	GDEV_PRINT("Closed gdev%d\n", h->dev_id);
 
 	FREE(h);
 

common/gdev_device.c

@@ -47,17 +47,17 @@ void __gdev_init_device(struct gdev_device *gdev, int id)
 	gdev->chipset = 0;
 	gdev->com_bw = 100;
 	gdev->mem_bw = 100;
+	gdev->mem_sh = 100;
 	gdev->com_bw_used = 0;
 	gdev->mem_bw_used = 0;
-	gdev->mem_sh = 100;
 	gdev->period = 0;
 	gdev->com_time = 0;
 	gdev->mem_time = 0;
 	gdev->swap = NULL;
 	gdev->sched_com_thread = NULL;
 	gdev->sched_mem_thread = NULL;
-	gdev->se_com_current = NULL;
-	gdev->se_mem_current = NULL;
+	gdev->current_com = NULL;
+	gdev->current_mem = NULL;
 	gdev->parent = NULL;
 	gdev->priv = NULL;
 	gdev_time_us(&gdev->credit_com, 0);
@@ -103,15 +103,18 @@ void gdev_exit_device(struct gdev_device *gdev)
 }
 
 /* initialize the virtual device information. */
-int gdev_init_virtual_device(struct gdev_device *gdev, int id, struct gdev_device *phys)
+int gdev_init_virtual_device(struct gdev_device *gdev, int id, uint32_t weight, struct gdev_device *phys)
 {
 	__gdev_init_device(gdev, id);
 	gdev->period = GDEV_PERIOD_DEFAULT;
 	gdev->parent = phys;
 	gdev->priv = phys->priv;
 	gdev->compute = phys->compute;
-	gdev->mem_size = phys->mem_size;
-	gdev->dma_mem_size = phys->dma_mem_size;
+	gdev->mem_size = phys->mem_size * weight / 100;
+	gdev->dma_mem_size = phys->dma_mem_size * weight / 100;
+	gdev->com_bw = weight;
+	gdev->mem_bw = weight;
+	gdev->mem_sh = weight;
 	gdev->chipset = phys->chipset;
 
 	/* create the swap memory object, if configured, for the virtual device. */

common/gdev_device.h

@@ -65,8 +65,8 @@ struct gdev_device {
 	void *compute; /* private set of compute functions */
 	void *sched_com_thread; /* compute scheduler thread */
 	void *sched_mem_thread; /* memory scheduler thread */
-	struct gdev_sched_entity *se_com_current; /* compute scheduling entity */
-	struct gdev_sched_entity *se_mem_current; /* memory scheduling entity */
+	void *current_com; /* current compute execution entity */
+	void *current_mem; /* current memory transfer entity */
 	struct gdev_device *parent; /* only for virtual devices */
 	struct gdev_list list_entry_com; /* entry to active compute list */
 	struct gdev_list list_entry_mem; /* entry to active memory list */
@@ -84,7 +84,7 @@ struct gdev_device {
 int gdev_init_device(struct gdev_device *gdev, int id, void *priv);
 void gdev_exit_device(struct gdev_device *gdev);
 
-int gdev_init_virtual_device(struct gdev_device *gdev, int id, struct gdev_device *parent);
+int gdev_init_virtual_device(struct gdev_device *gdev, int id, uint32_t weight, struct gdev_device *parent);
 void gdev_exit_virtual_device(struct gdev_device*);
 
 extern int gdev_count;

common/gdev_list.h

@@ -52,8 +52,7 @@ static inline void gdev_list_add_next(struct gdev_list *entry, struct gdev_list
 	struct gdev_list *next = pos->next;
 
 	entry->next = next;
-	if (next)
-		next->prev = entry;
+	next->prev = entry;
 	entry->prev = pos;
 	pos->next = entry;
 }
@@ -63,8 +62,7 @@ static inline void gdev_list_add_prev(struct gdev_list *entry, struct gdev_list
 	struct gdev_list *prev = pos->prev;
 
 	entry->prev = prev;
-	if (prev)
-		prev->next = entry;
+	prev->next = entry;
 	entry->next = pos;
 	pos->prev = entry;
 }
@@ -85,23 +83,20 @@ static inline void gdev_list_del(struct gdev_list *entry)
 	struct gdev_list *prev = entry->prev;
 
 	/* if prev is null, @entry points to the head, hence something wrong. */
-	if (prev) {
-		prev->next = next;
-		if (next)
-			next->prev = prev;
-		entry->next = entry->prev = NULL;
-	}
+	prev->next = next;
+	next->prev = prev;
+	entry->next = entry->prev = entry;
 }
 
 static inline int gdev_list_empty(struct gdev_list *entry)
 {
-	return entry->next == entry->prev;
+	return (entry->next == entry->prev) && (entry->next == entry);
 }
 
 static inline struct gdev_list *gdev_list_head(struct gdev_list *head)
 {
 	/* head->next is the actual head of the list. */
-	return head ? head->next : NULL;
+	return (head && !gdev_list_empty(head)) ? head->next : NULL;
 }
 
 static inline void *gdev_list_container(struct gdev_list *entry)

common/gdev_sched.c

@@ -165,7 +165,7 @@ void gdev_schedule_compute(struct gdev_sched_entity *se)
 
 	/* local compute scheduler. */
 	gdev_lock(&gdev->sched_com_lock);
-	if (gdev->se_com_current && gdev->se_com_current != se) {
+	if (gdev->current_com && gdev->current_com != se) {
 		/* enqueue the scheduling entity to the compute queue. */
 		__gdev_enqueue_compute(gdev, se);
 		gdev_unlock(&gdev->sched_com_lock);
@@ -183,7 +183,7 @@ void gdev_schedule_compute(struct gdev_sched_entity *se)
 			gdev_time_stamp(&se->last_tick_com);
 		}
 		se->launch_instances++;
-		gdev->se_com_current = se;
+		gdev->current_com = (void*)se;
 		gdev_unlock(&gdev->sched_com_lock);
 	}
 }
@@ -194,11 +194,12 @@ void gdev_schedule_compute(struct gdev_sched_entity *se)
  */
 void gdev_select_next_compute(struct gdev_device *gdev)
 {
-	struct gdev_sched_entity *se, *next;
+	struct gdev_sched_entity *se;
+	struct gdev_device *next;
 	struct gdev_time now, exec;
 
 	gdev_lock(&gdev->sched_com_lock);
-	se = gdev->se_com_current;
+	se = (struct gdev_sched_entity *)gdev->current_com;
 	if (!se) {
 		gdev_unlock(&gdev->sched_com_lock);
 		GDEV_PRINT("Invalid scheduling entity on Gdev#%d\n", gdev->id);
@@ -207,30 +208,50 @@ void gdev_select_next_compute(struct gdev_device *gdev)
 
 	/* record the end time (update on multiple launches too). */
 	gdev_time_stamp(&now);
-	/* account for the execution time. */
+	/* aquire the execution time. */
 	gdev_time_sub(&exec, &now, &se->last_tick_com);
 
 	se->launch_instances--;
 	if (se->launch_instances == 0) {
-		/* select the next device to be scheduled. */
-		gdev = gdev_vsched->select_next_compute(gdev);
-
-		/* select the next context to be scheduled. */
-		next = gdev_list_container(gdev_list_head(&gdev->sched_com_list));
-		/* remove it from the waiting list. */
-		if (next)
-			__gdev_dequeue_compute(next);
-		gdev->se_com_current = NULL; /* null clear once. */
+		/* account for the credit. */
+		gdev_time_sub(&gdev->credit_com, &gdev->credit_com, &exec);
+		/* accumulate the computation time. */
+		gdev->com_time += gdev_time_to_us(&exec);
+
+		/* select the next context to be scheduled.
+		   now don't reference the previous entity by se. */
+		se = gdev_list_container(gdev_list_head(&gdev->sched_com_list));
+		/* setting the next entity here prevents lower-priority contexts 
+		   arriving in gdev_schedule_compute() from being dispatched onto
+		   the device. note that se = NULL could happen. */
+		gdev->current_com = (void*)se; 
 		gdev_unlock(&gdev->sched_com_lock);
 
-		/* wake up the next context! */
-		if (next) {
-			/* could be enforced when awakened. */
-			gdev_sched_wakeup(next->task);
-		}
+		printk("gdev%d->credit_com = %s%lu\n", gdev->id, 
+			   gdev->credit_com.neg ? "-" : "",
+			   gdev_time_to_us(&gdev->credit_com));
 
-		/* accumulate the computation time on the device. */
-		gdev->com_time += gdev_time_to_us(&exec);
+		/* select the next device to be scheduled. */
+		next = gdev_vsched->select_next_compute(gdev);
+		if (!next)
+			return;
+
+		gdev_lock(&next->sched_com_lock);
+		/* if the virtual device needs to be switched, change the next
+		   scheduling entity to be scheduled also needs to be changed. */
+		if (next != gdev)
+			se = gdev_list_container(gdev_list_head(&next->sched_com_list));
+
+		/* now remove the scheduling entity from the waiting list, and wake 
+		   up the corresponding task. */
+		if (se) {
+			__gdev_dequeue_compute(se);
+			gdev_unlock(&next->sched_com_lock);
+
+			gdev_sched_wakeup(se->task); 
+		}
+		else
+			gdev_unlock(&next->sched_com_lock);
 	}
 	else
 		gdev_unlock(&gdev->sched_com_lock);

common/gdev_vsched_credit.c

@@ -36,7 +36,7 @@ static void gdev_vsched_credit_schedule_compute(struct gdev_sched_entity *se)
 
 resched:
 	gdev_lock(&phys->sched_com_lock);
-	if (phys->se_com_current && phys->se_com_current != se) {
+	if (phys->current_com && phys->current_com != gdev) {
 		/* insert the scheduling entity to its local priority-ordered list. */
 		gdev_lock_nested(&gdev->sched_com_lock);
 		__gdev_enqueue_compute(gdev, se);
@@ -50,7 +50,7 @@ static void gdev_vsched_credit_schedule_compute(struct gdev_sched_entity *se)
 		goto resched;
 	}
 	else {
-		phys->se_com_current = se;
+		phys->current_com = (void *)gdev;
 		gdev_unlock(&phys->sched_com_lock);
 	}
 }
@@ -59,47 +59,31 @@ static struct gdev_device *gdev_vsched_credit_select_next_compute(struct gdev_de
 {
 	struct gdev_device *phys = gdev->parent;
 	struct gdev_device *next;
-	struct gdev_sched_entity *se;
-	struct gdev_time now, zero;
+	struct gdev_time zero;
 
 	if (!phys)
 		return gdev;
 
-	/* account for the computation time. */
-	se = gdev->se_com_current;
-	gdev_time_stamp(&now);
-	gdev_time_sub(&gdev->credit_com, &now, &se->last_tick_com);
-
-	/* select the next device. */
 	gdev_lock(&phys->sched_com_lock);
 
 	/* if the credit is exhausted, reinsert the device. */
 	gdev_time_us(&zero, 0);
 	if (gdev_time_le(&gdev->credit_com, &zero)) {
 		gdev_list_del(&gdev->list_entry_com);
-		gdev_list_add(&gdev->list_entry_com, &phys->sched_com_list);
+		gdev_list_add_tail(&gdev->list_entry_com, &phys->sched_com_list);
 	}
 
 	gdev_list_for_each(next, &phys->sched_com_list, list_entry_com) {
-		/* if the current device is found first as an available device, break
-		   the search loop. note that gdev->sched_com_lock is alreay locked. */
-		if (next == gdev)
-			goto device_not_switched;
-		else {
-			gdev_lock_nested(&next->sched_com_lock);
-			if (!gdev_list_empty(&next->sched_com_list)) {
-				/* unlock the current device. the next device will be 
-				   unlocked after this function returns. */
-				gdev_unlock_nested(&gdev->sched_com_lock);
-				goto device_switched;
-			}
+		gdev_lock_nested(&next->sched_com_lock);
+		if (!gdev_list_empty(&next->sched_com_list)) {
 			gdev_unlock_nested(&next->sched_com_lock);
+			goto device_switched;
 		}
+		gdev_unlock_nested(&next->sched_com_lock);
 	}
-device_not_switched:
-	next = gdev;
+	next = NULL;
 device_switched:
-	phys->se_com_current = NULL; /* null clear */
+	phys->current_com = (void*)next; /* could be null */
 	gdev_unlock(&phys->sched_com_lock);
 
 	return next;
@@ -113,9 +97,12 @@ static void gdev_vsched_credit_replenish_compute(struct gdev_device *gdev)
 	gdev_time_add(&gdev->credit_com, &gdev->credit_com, &credit);
 	/* when the credit exceeds the threshold, all credits taken away. */
 	gdev_time_us(&threshold, GDEV_CREDIT_INACTIVE_THRESHOLD);
-	if (gdev_time_gt(&gdev->credit_com, &threshold)) {
+	if (gdev_time_gt(&gdev->credit_com, &threshold))
+		gdev_time_us(&gdev->credit_com, 0);
+	/* when the credit exceeds the threshold in negative, even it. */
+	threshold.neg = 1;
+	if (gdev_time_lt(&gdev->credit_com, &threshold))
 		gdev_time_us(&gdev->credit_com, 0);
-	}
 }
 
 static struct gdev_vsched_policy gdev_vsched_credit = {

cuda/libcuda/gdev_cuda_util.h

@@ -55,15 +55,15 @@ typedef FILE file_t;
 
 static inline int __gdev_get_device_count(void)
 {
-	char fname[64] = "/proc/gdev/device_count";
+	char fname[256] = "/proc/gdev/virtual_device_count";
 	char buf[16];
 	int minor = 0;
 	FILE *fp;
 
-	if (!(fp = FOPEN(fname)))
+	if (!(fp = fopen(fname, "r")))
 		return 0;
-	FREAD(buf, 15, fp);
-	FCLOSE(fp);
+	fgets(buf, 16, fp);
+	fclose(fp);
 
 	sscanf(buf, "%d", &minor);
 

driver/gdev/gdev_conf.h

@@ -42,7 +42,15 @@
 
 #define GDEV_MEMCPY_IORW_LIMIT 0x400 /* bytes */
 
-#define GDEV_VIRTUAL_DEVICE_COUNT 4 /* # of virtual devices */
+#define GDEV_PHYSICAL_DEVICE_MAX_COUNT 8 /* # of physical devices */
+#define GDEV0_VIRTUAL_DEVICE_COUNT 4 /* # of virtual devices */
+#define GDEV1_VIRTUAL_DEVICE_COUNT 0 /* # of virtual devices */
+#define GDEV2_VIRTUAL_DEVICE_COUNT 0 /* # of virtual devices */
+#define GDEV3_VIRTUAL_DEVICE_COUNT 0 /* # of virtual devices */
+#define GDEV4_VIRTUAL_DEVICE_COUNT 0 /* # of virtual devices */
+#define GDEV5_VIRTUAL_DEVICE_COUNT 0 /* # of virtual devices */
+#define GDEV6_VIRTUAL_DEVICE_COUNT 0 /* # of virtual devices */
+#define GDEV7_VIRTUAL_DEVICE_COUNT 0 /* # of virtual devices */
 
 // #define GDEV_DEBUG_PRINT