Improve cpu_idle when PM is disabled

Split cpu_idle() into cpu_idle_delay() and cpu_idle_job() rather than
requesting the idle type as a function argument. Have those functions
provide a default polling (non-PM) implentation which spin at the
lowest SMT priority.

This moves all the decrementer delay code into the CPU idle code rather
than the caller.

Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Stewart Smith <stewart@linux.vnet.ibm.com>

core/cpu.c

@@ -286,22 +286,18 @@ void cpu_process_jobs(void)
 	unlock(&cpu->job_lock);
 }
 
-static void cpu_idle_default(enum cpu_wake_cause wake_on __unused)
-{
-	/* Maybe do something better for simulators ? */
-	cpu_relax();
-	cpu_relax();
-	cpu_relax();
-	cpu_relax();
-}
+enum cpu_wake_cause {
+	cpu_wake_on_job,
+	cpu_wake_on_dec,
+};
 
 static void cpu_idle_p8(enum cpu_wake_cause wake_on)
 {
 	uint64_t lpcr = mfspr(SPR_LPCR) & ~SPR_LPCR_P8_PECE;
 	struct cpu_thread *cpu = this_cpu();
 
 	if (!pm_enabled) {
-		cpu_idle_default(wake_on);
+		prlog_once(PR_DEBUG, "cpu_idle_p8 called pm disabled\n");
 		return;
 	}
 
@@ -373,18 +369,60 @@ void cpu_set_pm_enable(bool enabled)
 	}
 }
 
-void cpu_idle(enum cpu_wake_cause wake_on)
+static void cpu_idle_pm(enum cpu_wake_cause wake_on)
 {
 	switch(proc_gen) {
 	case proc_gen_p8:
 		cpu_idle_p8(wake_on);
 		break;
 	default:
-		cpu_idle_default(wake_on);
+		prlog_once(PR_DEBUG, "cpu_idle_pm called with bad processor type\n");
 		break;
 	}
 }
 
+void cpu_idle_job(void)
+{
+	if (pm_enabled) {
+		cpu_idle_pm(cpu_wake_on_job);
+	} else {
+		struct cpu_thread *cpu = this_cpu();
+
+		smt_lowest();
+		/* Check for jobs again */
+		while (!cpu_check_jobs(cpu))
+			barrier();
+		smt_medium();
+	}
+}
+
+void cpu_idle_delay(unsigned long delay, unsigned long min_pm)
+{
+	unsigned long now = mftb();
+	unsigned long end = now + delay;
+
+	if (pm_enabled && delay > min_pm) {
+		for (;;) {
+			if (delay >= 0x7fffffff)
+				delay = 0x7fffffff;
+			mtspr(SPR_DEC, delay);
+
+			cpu_idle_pm(cpu_wake_on_dec);
+
+			now = mftb();
+			if (tb_compare(now, end) == TB_AAFTERB)
+				break;
+
+			delay = end - now;
+		}
+	} else {
+		smt_lowest();
+		while (tb_compare(mftb(), end) != TB_AAFTERB)
+			barrier();
+		smt_medium();
+	}
+}
+
 void cpu_process_local_jobs(void)
 {
 	struct cpu_thread *cpu = first_available_cpu();

core/init.c

@@ -1068,9 +1068,9 @@ void __noreturn __secondary_cpu_entry(void)
 	/* Wait for work to do */
 	while(true) {
 		if (cpu_check_jobs(cpu))
-		    cpu_process_jobs();
+			cpu_process_jobs();
 		else
-		    cpu_idle(cpu_wake_on_job);
+			cpu_idle_job();
 	}
 }
 

core/timebase.c

@@ -24,16 +24,18 @@ unsigned long tb_hz = 512000000;
 
 static void time_wait_poll(unsigned long duration)
 {
-	unsigned long remaining = duration;
-	unsigned long end = mftb() + duration;
+	unsigned long now = mftb();
+	unsigned long end = now + duration;
 	unsigned long period = msecs_to_tb(5);
 
 	if (this_cpu()->tb_invalid) {
 		cpu_relax();
 		return;
 	}
 
-	while (tb_compare(mftb(), end) != TB_AAFTERB) {
+	while (tb_compare(now, end) != TB_AAFTERB) {
+		unsigned long remaining = end - now;
+
 		/* Call pollers periodically but not continually to avoid
 		 * bouncing cachelines due to lock contention. */
 		if (remaining >= period) {
@@ -43,7 +45,7 @@ static void time_wait_poll(unsigned long duration)
 		} else
 			time_wait_nopoll(remaining);
 
-		cpu_relax();
+		now = mftb();
 	}
 }
 
@@ -64,28 +66,14 @@ void time_wait(unsigned long duration)
 
 void time_wait_nopoll(unsigned long duration)
 {
-	unsigned long end = mftb() + duration;
-	unsigned long min = usecs_to_tb(10);
+	unsigned long min_sleep = usecs_to_tb(10);
 
 	if (this_cpu()->tb_invalid) {
 		cpu_relax();
 		return;
 	}
 
-	for (;;) {
-		uint64_t delay, tb = mftb();
-
-		if (tb_compare(tb, end) == TB_AAFTERB)
-			break;
-		delay = end - tb;
-		if (delay >= 0x7fffffff)
-			delay = 0x7fffffff;
-		if (delay >= min) {
-			mtspr(SPR_DEC, delay);
-			cpu_idle(cpu_wake_on_dec);
-		} else
-			cpu_relax();
-	}
+	cpu_idle_delay(duration, min_sleep);
 }
 
 void time_wait_ms(unsigned long ms)

include/cpu.h

@@ -43,11 +43,6 @@ enum cpu_thread_state {
 struct cpu_job;
 struct xive_cpu_state;
 
-enum cpu_wake_cause {
-	cpu_wake_on_job,
-	cpu_wake_on_dec,
-};
-
 struct cpu_thread {
 	uint32_t			pir;
 	uint32_t			server_no;
@@ -274,6 +269,7 @@ static inline void cpu_give_self_os(void)
 extern unsigned long __attrconst cpu_stack_bottom(unsigned int pir);
 extern unsigned long __attrconst cpu_stack_top(unsigned int pir);
 
-extern void cpu_idle(enum cpu_wake_cause wake_on);
+extern void cpu_idle_job(void);
+extern void cpu_idle_delay(unsigned long delay, unsigned long min_pm);
 
 #endif /* __CPU_H */