sched/core: Optimize the process of picking the max prio task for the…

… core

When we pick the max prio task for the core in the case of
sched_core_enabled(), if there's a task with a higher prio sched
class in the runqueue of a SMT, it's not necessary for other SMTs
to traverse lower prio sched classes. So we can change the traversal
order: for each sched class, if there exists a max prio task among
the core, pick it and break the loop.

To compare the prio of the tasks with the same sched class, we
introduce sched_class::prio_less().

Signed-off-by: Cruz Zhao <CruzZhao@linux.alibaba.com>

kernel/sched/core.c

@@ -188,7 +188,7 @@ static inline bool prio_less(struct task_struct *a, struct task_struct *b, bool
 		return !dl_time_before(a->dl.deadline, b->dl.deadline);
 
 	if (pa == MAX_RT_PRIO + MAX_NICE)	/* fair */
-		return cfs_prio_less(a, b, in_fi);
+		return fair_sched_class.prio_less(a, b, in_fi);
 
 	return false;
 }
@@ -5869,6 +5869,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 	int i, cpu, occ = 0;
 	struct rq *rq_i;
 	bool need_sync;
+	struct sched_class *class;
 
 	if (!sched_core_enabled(rq))
 		return __pick_next_task(rq, prev, rf);
@@ -5961,12 +5962,6 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 		}
 	}
 
-	/*
-	 * For each thread: do the regular task pick and find the max prio task
-	 * amongst them.
-	 *
-	 * Tie-break prio towards the current CPU
-	 */
 	for_each_cpu_wrap(i, smt_mask, cpu) {
 		rq_i = cpu_rq(i);
 
@@ -5977,12 +5972,28 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 		 */
 		if (i != cpu && (rq_i != rq->core || !core_clock_updated))
 			update_rq_clock(rq_i);
+	}
 
-		p = rq_i->core_pick = pick_task(rq_i);
-		if (!max || prio_less(max, p, fi_before))
-			max = p;
+	/*
+	 * For each thread: do the regular task pick and find the max prio task
+	 * amongst them.
+	 *
+	 * Tie-break prio towards the current CPU
+	 */
+	for_each_class(class) {
+		for_each_cpu_wrap(i, smt_mask, cpu) {
+			rq_i = cpu_rq(i);
+			p = rq_i->core_pick = class->pick_task(rq_i);
+			if (!max || (p && class->prio_less(max, p, fi_before)))
+				max = p;
+		}
+		if (max)
+			break;
 	}
 
+	if (!max)
+		BUG();
+
 	cookie = rq->core->core_cookie = max->core_cookie;
 
 	/*
@@ -5993,7 +6004,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
 		rq_i = cpu_rq(i);
 		p = rq_i->core_pick;
 
-		if (!cookie_equals(p, cookie)) {
+		if (!p || !cookie_equals(p, cookie)) {
 			p = NULL;
 			if (cookie)
 				p = sched_core_find(rq_i, cookie);

kernel/sched/deadline.c

@@ -1942,6 +1942,7 @@ static int balance_dl(struct rq *rq, struct task_struct *p, struct rq_flags *rf)
 
 	return sched_stop_runnable(rq) || sched_dl_runnable(rq);
 }
+
 #endif /* CONFIG_SMP */
 
 /*
@@ -2054,6 +2055,13 @@ static void put_prev_task_dl(struct rq *rq, struct task_struct *p)
 		enqueue_pushable_dl_task(rq, p);
 }
 
+#ifdef CONFIG_SCHED_CORE
+static bool prio_less_dl(struct task_struct *a, struct task_struct *b, bool in_fi)
+{
+	return !dl_time_before(a->dl.deadline, b->dl.deadline);
+}
+#endif
+
 /*
  * scheduler tick hitting a task of our scheduling class.
  *
@@ -2704,6 +2712,9 @@ DEFINE_SCHED_CLASS(dl) = {
 	.pick_next_task		= pick_next_task_dl,
 	.put_prev_task		= put_prev_task_dl,
 	.set_next_task		= set_next_task_dl,
+#ifdef CONFIG_SCHED_CORE
+	.prio_less		= prio_less_dl,
+#endif
 
 #ifdef CONFIG_SMP
 	.balance		= balance_dl,

kernel/sched/fair.c

@@ -11347,7 +11347,7 @@ void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi)
 	se_fi_update(se, rq->core->core_forceidle_seq, in_fi);
 }
 
-bool cfs_prio_less(struct task_struct *a, struct task_struct *b, bool in_fi)
+static bool prio_less_fair(struct task_struct *a, struct task_struct *b, bool in_fi)
 {
 	struct rq *rq = task_rq(a);
 	struct sched_entity *sea = &a->se;
@@ -11990,6 +11990,9 @@ DEFINE_SCHED_CLASS(fair) = {
 	.pick_next_task		= __pick_next_task_fair,
 	.put_prev_task		= put_prev_task_fair,
 	.set_next_task          = set_next_task_fair,
+#ifdef CONFIG_SCHED_CORE
+	.prio_less		= prio_less_fair,
+#endif
 
 #ifdef CONFIG_SMP
 	.balance		= balance_fair,

kernel/sched/idle.c

@@ -436,6 +436,13 @@ static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool fir
 	schedstat_inc(rq->sched_goidle);
 }
 
+#ifdef CONFIG_SCHED_CORE
+static bool prio_less_idle(struct task_struct *a, struct task_struct *b, bool in_fi)
+{
+	return false;
+}
+#endif
+
 #ifdef CONFIG_SMP
 static struct task_struct *pick_task_idle(struct rq *rq)
 {
@@ -507,6 +514,9 @@ DEFINE_SCHED_CLASS(idle) = {
 	.pick_next_task		= pick_next_task_idle,
 	.put_prev_task		= put_prev_task_idle,
 	.set_next_task          = set_next_task_idle,
+#ifdef CONFIG_SCHED_CORE
+	.prio_less		= prio_less_idle,
+#endif
 
 #ifdef CONFIG_SMP
 	.balance		= balance_idle,

kernel/sched/rt.c

@@ -1838,6 +1838,13 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p)
 		enqueue_pushable_task(rq, p);
 }
 
+static bool prio_less_rt(struct task_struct *a, struct task_struct *b, bool in_fi)
+{
+	int pa = rt_prio(a->prio), pb = rt_prio(b->prio);
+
+	return -pa < -pb;
+}
+
 #ifdef CONFIG_SMP
 
 /* Only try algorithms three times */
@@ -2685,6 +2692,9 @@ DEFINE_SCHED_CLASS(rt) = {
 	.pick_next_task		= pick_next_task_rt,
 	.put_prev_task		= put_prev_task_rt,
 	.set_next_task          = set_next_task_rt,
+#ifdef CONFIG_SCHED_CORE
+	.prio_less		= prio_less_rt,
+#endif
 
 #ifdef CONFIG_SMP
 	.balance		= balance_rt,

kernel/sched/sched.h

@@ -1218,8 +1218,6 @@ static inline raw_spinlock_t *__rq_lockp(struct rq *rq)
 	return &rq->__lock;
 }
 
-bool cfs_prio_less(struct task_struct *a, struct task_struct *b, bool fi);
-
 /*
  * Helpers to check if the CPU's core cookie matches with the task's cookie
  * when core scheduling is enabled.
@@ -2155,6 +2153,9 @@ struct sched_class {
 
 	void (*put_prev_task)(struct rq *rq, struct task_struct *p);
 	void (*set_next_task)(struct rq *rq, struct task_struct *p, bool first);
+#ifdef CONFIG_SCHED_CORE
+	bool prio_less(struct task_struct *a, struct task_struct *b, bool in_fi);
+#endif
 
 #ifdef CONFIG_SMP
 	int (*balance)(struct rq *rq, struct task_struct *prev, struct rq_flags *rf);

kernel/sched/stop_task.c

@@ -84,6 +84,13 @@ static void put_prev_task_stop(struct rq *rq, struct task_struct *prev)
 	update_current_exec_runtime(curr, now, delta_exec);
 }
 
+#ifdef CONFIG_SCHED_CORE
+static bool prio_less_stop(struct task_struct *a, struct task_struct *b, bool in_fi)
+{
+	return false;
+}
+#endif
+
 /*
  * scheduler tick hitting a task of our scheduling class.
  *
@@ -125,6 +132,9 @@ DEFINE_SCHED_CLASS(stop) = {
 	.pick_next_task		= pick_next_task_stop,
 	.put_prev_task		= put_prev_task_stop,
 	.set_next_task          = set_next_task_stop,
+#ifdef CONFIG_SCHED_CORE
+	.prio_less		= prio_less_stop,
+#endif
 
 #ifdef CONFIG_SMP
 	.balance		= balance_stop,