Permalink
Browse files

gdev: added a priority-based scheduler

test: added CUDA loop repeated test
  • Loading branch information...
Shinpei Kato
Shinpei Kato committed Jan 8, 2012
1 parent 246ecef commit 5e75425a897a7221d6fadcba46b14acb96ba43dc
View
@@ -268,13 +268,18 @@ static int __gmemcpy_to_device
gdev_ctx_t *ctx = h->ctx;
gdev_mem_t *mem = gdev_mem_lookup(vas, dst_addr, GDEV_MEM_DEVICE);
gdev_mem_t **dma_mem = h->dma_mem;
+ struct gdev_sched_entity *se = h->se;
uint32_t ch_size = h->chunk_size;
int p_count = h->pipeline_count;
int ret;
if (!mem)
return -ENOENT;
+#ifndef GDEV_SCHEDULER_DISABLED
+ gdev_schedule_memcpy(se);
+#endif
+
gdev_mem_lock(mem);
gdev_shm_evict_conflict(ctx, mem); /* evict conflicting data. */
ret = __gmemcpy_to_device_locked(ctx, dst_addr, src_buf, size, async,
@@ -456,13 +461,18 @@ static int __gmemcpy_from_device
gdev_ctx_t *ctx = h->ctx;
gdev_mem_t *mem = gdev_mem_lookup(vas, src_addr, GDEV_MEM_DEVICE);
gdev_mem_t **dma_mem = h->dma_mem;
+ struct gdev_sched_entity *se = h->se;
uint32_t ch_size = h->chunk_size;
int p_count = h->pipeline_count;
int ret;
if (!mem)
return -ENOENT;
+#ifndef GDEV_SCHEDULER_DISABLED
+ gdev_schedule_memcpy(se);
+#endif
+
gdev_mem_lock(mem);
gdev_shm_retrieve_swap(ctx, mem); /* retrieve data swapped. */
ret = __gmemcpy_from_device_locked(ctx, dst_buf, src_addr, size, async,
@@ -881,7 +891,9 @@ int glaunch(struct gdev_handle *h, struct gdev_kernel *kernel, uint32_t *id)
gdev_ctx_t *ctx = h->ctx;
struct gdev_sched_entity *se = h->se;
+#ifndef GDEV_SCHEDULER_DISABLED
gdev_schedule_launch(se);
+#endif
gdev_mem_lock_all(vas);
gdev_shm_retrieve_swap_all(ctx, vas); /* get all data swapped back! */
View
@@ -35,25 +35,43 @@ int gdev_vcount = 0; /* # of virtual devices. */
struct gdev_device *gdevs = NULL; /* physical devices */
struct gdev_device *gdev_vds = NULL; /* virtual devices */
-/* initialize the physical device information. */
-int gdev_init_device(struct gdev_device *gdev, int minor, void *priv)
+void __gdev_init_device(struct gdev_device *gdev, int id)
{
- gdev->id = minor;
+ gdev->id = id;
gdev->users = 0;
gdev->mem_size = 0;
gdev->mem_used = 0;
gdev->dma_mem_size = 0;
gdev->dma_mem_used = 0;
- gdev->proc_util = 100; /* 100% */
- gdev->mem_util = 100; /* 100% */
+ gdev->chipset = 0;
+ gdev->com_bw = 0;
+ gdev->mem_bw = 0;
+ gdev->mem_sh = 0;
gdev->swap = NULL;
- gdev->sched_thread = NULL;
+ gdev->sched_com_thread = NULL;
+ gdev->sched_mem_thread = NULL;
+ gdev->se_com_current = NULL;
+ gdev->se_mem_current = NULL;
gdev->parent = NULL;
- gdev->priv = priv; /* this must be set before calls to gdev_query(). */
- gdev_list_init(&gdev->vas_list, NULL); /* VAS list. */
- gdev_list_init(&gdev->shm_list, NULL); /* shared memory list. */
+ gdev->priv = NULL;
+ gdev_list_init(&gdev->sched_com_list, NULL);
+ gdev_list_init(&gdev->sched_mem_list, NULL);
+ gdev_list_init(&gdev->vas_list, NULL);
+ gdev_list_init(&gdev->shm_list, NULL);
+ gdev_lock_init(&gdev->sched_com_lock);
+ gdev_lock_init(&gdev->sched_mem_lock);
gdev_lock_init(&gdev->vas_lock);
gdev_mutex_init(&gdev->shm_mutex);
+}
+
+/* initialize the physical device information. */
+int gdev_init_device(struct gdev_device *gdev, int id, void *priv)
+{
+ __gdev_init_device(gdev, id);
+ gdev->com_bw = 100; /* 100% */
+ gdev->mem_bw = 100; /* 100% */
+ gdev->mem_sh = 100; /* 100% */
+ gdev->priv = priv; /* this must be set before calls to gdev_query(). */
/* architecture-dependent chipset.
this call must be prior to the following. */
@@ -80,26 +98,18 @@ void gdev_exit_device(struct gdev_device *gdev)
}
/* initialize the virtual device information. */
-int gdev_init_virtual_device
-(struct gdev_device *gdev, int id, uint32_t proc_util, uint32_t mem_util,
- struct gdev_device *phys)
+int gdev_init_virtual_device(struct gdev_device *gdev, int id, uint32_t com_bw, uint32_t mem_bw, uint32_t mem_sh, struct gdev_device *phys)
{
- gdev->id = id;
- gdev->users = 0;
- gdev->proc_util = proc_util;
- gdev->mem_util = mem_util;
- gdev->swap = NULL;
- gdev->sched_thread = NULL;
+ __gdev_init_device(gdev, id);
+ gdev->com_bw = com_bw;
+ gdev->mem_bw = mem_bw;
+ gdev->mem_sh = mem_sh;
gdev->parent = phys;
gdev->priv = phys->priv;
gdev->compute = phys->compute;
- gdev->mem_size = phys->mem_size * mem_util / 100;
- gdev->dma_mem_size = phys->dma_mem_size * mem_util / 100;
+ gdev->mem_size = phys->mem_size * mem_sh / 100;
+ gdev->dma_mem_size = phys->dma_mem_size * mem_sh / 100;
gdev->chipset = phys->chipset;
- gdev_list_init(&gdev->vas_list, NULL); /* VAS list. */
- gdev_list_init(&gdev->shm_list, NULL); /* shared memory list. */
- gdev_lock_init(&gdev->vas_lock);
- gdev_mutex_init(&gdev->shm_mutex);
/* create the swap memory object, if configured, for the virtual device. */
if (GDEV_SWAP_MEM_SIZE > 0) {
View
@@ -33,6 +33,13 @@
#include "gdev_list.h"
#include "gdev_system.h"
+/**
+ * generic subchannel definitions
+ */
+#define GDEV_SUBCH_LAUNCH 1
+#define GDEV_SUBCH_MEMCPY 2
+#define GDEV_SUBCH_MEMCPY_ASYNC 3
+
/**
* Gdev device struct:
*/
@@ -44,23 +51,31 @@ struct gdev_device {
uint64_t mem_used;
uint64_t dma_mem_size;
uint64_t dma_mem_used;
- uint32_t proc_util; /* available processor utilization */
- uint32_t mem_util; /* available memory utilization */
+ uint32_t com_bw; /* available compute bandwidth */
+ uint32_t mem_bw; /* available memory bandwidth */
+ uint32_t mem_sh; /* available memory space share */
void *priv; /* private device object */
void *compute; /* private set of compute functions */
- void *sched_thread; /* scheduler thread */
+ void *sched_com_thread; /* compute scheduler thread */
+ void *sched_mem_thread; /* memory scheduler thread */
+ struct gdev_sched_entity *se_com_current; /* compute scheduling entity */
+ struct gdev_sched_entity *se_mem_current; /* memory scheduling entity */
struct gdev_device *parent; /* only for virtual devices */
+ struct gdev_list sched_com_list; /* wait list for compute scheduling */
+ struct gdev_list sched_mem_list; /* wait list for memory scheduling */
struct gdev_list vas_list; /* list of VASes allocated to this device */
- struct gdev_list shm_list; /* list of VASes allocated to this device */
+ struct gdev_list shm_list; /* list of shm users allocated to this device */
+ gdev_lock_t sched_com_lock;
+ gdev_lock_t sched_mem_lock;
gdev_lock_t vas_lock;
gdev_mutex_t shm_mutex;
gdev_mem_t *swap; /* reserved swap memory space */
};
-int gdev_init_device(struct gdev_device *gdev, int minor, void *priv);
+int gdev_init_device(struct gdev_device *gdev, int id, void *priv);
void gdev_exit_device(struct gdev_device *gdev);
-int gdev_init_virtual_device(struct gdev_device *gdev, int id, uint32_t proc_util, uint32_t mem_util, struct gdev_device *parent);
+int gdev_init_virtual_device(struct gdev_device *gdev, int id, uint32_t com_bw, uint32_t mem_bw, uint32_t mem_sh, struct gdev_device *parent);
void gdev_exit_virtual_device(struct gdev_device*);
extern int gdev_count;
View
@@ -47,8 +47,7 @@ static inline void gdev_list_init(struct gdev_list *entry, void *container)
entry->container = container;
}
-static inline void gdev_list_add
-(struct gdev_list *entry, struct gdev_list *head)
+static inline void gdev_list_add(struct gdev_list *entry, struct gdev_list *head)
{
struct gdev_list *next = head->next;
@@ -59,6 +58,28 @@ static inline void gdev_list_add
head->next = entry;
}
+static inline void gdev_list_add_next(struct gdev_list *entry, struct gdev_list *pos)
+{
+ struct gdev_list *next = pos->next;
+
+ entry->next = next;
+ if (next)
+ next->prev = entry;
+ entry->prev = pos;
+ pos->next = entry;
+}
+
+static inline void gdev_list_add_prev(struct gdev_list *entry, struct gdev_list *pos)
+{
+ struct gdev_list *prev = pos->prev;
+
+ entry->prev = prev;
+ if (prev)
+ prev->next = entry;
+ entry->next = pos;
+ pos->prev = entry;
+}
+
static inline void gdev_list_del(struct gdev_list *entry)
{
struct gdev_list *next = entry->next;
@@ -88,24 +109,9 @@ static inline void *gdev_list_container(struct gdev_list *entry)
return entry ? entry->container : NULL;
}
-#define gdev_list_for_each(p, list, entry) \
+#define gdev_list_for_each(p, list, entry_name) \
for (p = gdev_list_container(gdev_list_head(list)); \
p != NULL; \
- p = gdev_list_container((p)->entry.next))
-
-#define gdev_list_add_ordered(entry, head, member) \
- do { \
- struct gdev_list *p, *tail = head; \
- (entry)->next = (entry)->prev = NULL; \
- gdev_list_for_each(p, head) { \
- if ((entry)->member <= p->member) { \
- gdev_list_add(entry, p); \
- break; \
- } \
- tail = p; \
- } \
- if (gdev_list_empty(entry)) \
- gdev_list_add(entry, tail); \
- } while (0)
+ p = gdev_list_container((p)->entry_name.next))
#endif
View
@@ -35,10 +35,10 @@
#include "gdev_system.h"
#include "gdev_time.h"
-#define GDEV_SUBCH_COMPUTE 1
-#define GDEV_SUBCH_M2MF 2
-#define GDEV_SUBCH_PCOPY0 3
-#define GDEV_SUBCH_PCOPY1 4
+#define GDEV_SUBCH_COMPUTE GDEV_SUBCH_LAUNCH
+#define GDEV_SUBCH_M2MF GDEV_SUBCH_MEMCPY
+#define GDEV_SUBCH_PCOPY0 GDEV_SUBCH_MEMCPY_ASYNC
+#define GDEV_SUBCH_PCOPY1 (GDEV_SUBCH_MEMCPY_ASYNC + 1)
#define GDEV_FENCE_BUF_SIZE 0x10000 /* 64KB */
#define GDEV_FENCE_QUERY_SIZE 0x10 /* aligned with nvc0's query */
View
@@ -63,7 +63,12 @@ struct gdev_sched_entity *gdev_sched_entity_create(struct gdev_device *gdev, gde
se->gdev = gdev;
se->task = gdev_sched_get_current_task();
se->ctx = ctx;
+ se->prio = gdev_sched_get_static_prio(se->task);
se->rt_prio = GDEV_PRIO_DEFAULT;
+ se->launch_instances = 0;
+ se->memcpy_instances = 0;
+ gdev_list_init(&se->list_entry_com, (void*)se);
+ gdev_list_init(&se->list_entry_mem, (void*)se);
sched_entity_ptr[gdev_ctx_get_cid(ctx)] = se;
return se;
@@ -83,22 +88,89 @@ void gdev_sched_entity_destroy(struct gdev_sched_entity *se)
void gdev_schedule_launch(struct gdev_sched_entity *se)
{
struct gdev_device *gdev = se->gdev;
- //struct gdev_sched_entity *se_current = gdev->
+ struct gdev_sched_entity *p, *tail = NULL;
+resched:
+ gdev_lock(&gdev->sched_com_lock);
+ if (gdev->se_com_current && gdev->se_com_current != se) {
+ /* insert the scheduling entity to the priority-ordered list. */
+ gdev_list_for_each (p, &gdev->sched_com_list, list_entry_com) {
+ if (se->prio > p->prio) {
+ gdev_list_add_prev(&se->list_entry_com, &p->list_entry_com);
+ break;
+ }
+ tail = p;
+ }
+ if (gdev_list_empty(&se->list_entry_com)) {
+ if (tail)
+ gdev_list_add_next(&se->list_entry_com, &tail->list_entry_com);
+ else
+ gdev_list_add(&se->list_entry_com, &gdev->sched_com_list);
+ }
+ gdev_unlock(&gdev->sched_com_lock);
+
+ /* now the corresponding task will be suspended until some other tasks
+ will awaken it upon completions of their compute launches. */
+ gdev_sched_sleep();
+
+ goto resched;
+ }
+ else {
+ gdev->se_com_current = se;
+ se->launch_instances++;
+ gdev_unlock(&gdev->sched_com_lock);
+
+ printk("ctx %d excute\n", se->ctx->cid);
+ }
+}
+
+/**
+ * schedule next contexts for kernel launch.
+ * invoked upon the completion of preceding contexts.
+ */
+void gdev_schedule_launch_post(struct gdev_device *gdev)
+{
+ struct gdev_sched_entity *se, *next;
+
+ gdev_lock(&gdev->sched_com_lock);
+ se = gdev->se_com_current;
+ if (se) {
+ se->launch_instances--;
+ if (se->launch_instances == 0) {
+ /* select the next context. */
+ next = gdev_list_container(gdev_list_head(&gdev->sched_com_list));
+ /* remove it from the waiting list. */
+ if (next)
+ gdev_list_del(&next->list_entry_com);
+ /* now this is going to be the current context. */
+ gdev->se_com_current = next;
+ gdev_unlock(&gdev->sched_com_lock);
+ /* wake up the next context! */
+ if (next) {
+ gdev_sched_wakeup(next->task);
+ }
+ }
+ else
+ gdev_unlock(&gdev->sched_com_lock);
+ }
+ else
+ gdev_unlock(&gdev->sched_com_lock);
}
/**
* schedule memcpy-copy calls.
*/
void gdev_schedule_memcpy(struct gdev_sched_entity *se)
{
+ //gdev_schedule_launch(se);
}
/**
- * schedule next contexts.
+ * schedule next contexts for memory copy.
* invoked upon the completion of preceding contexts.
*/
-void gdev_schedule_invoked(int subc, uint32_t data)
+void gdev_schedule_memcpy_post(struct gdev_device *gdev)
{
+ //gdev_schedule_launch_post(gdev);
}
Oops, something went wrong.

0 comments on commit 5e75425

Please sign in to comment.