cpu: idle POWER9 power management implementation

Add pm idle support to POWER9. IPIs are implemented with doorbells.
POWER9 can use the EC=ESL=0 (lite) stop when sreset is not available.

EC=ESL=1 state with RL=3 is enabled when we have a sreset wakeup.

Deep idle states are not implemented.

Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Stewart Smith <stewart@linux.vnet.ibm.com>

asm/head.S

@@ -28,6 +28,8 @@
 #define PPC_INST_SLEEP		.long 0x4c0003a4
 #define PPC_INST_RVWINKLE	.long 0x4c0003e4
 
+#define PPC_INST_STOP		.long 0x4c0002e4
+
 #define GET_STACK(stack_reg,pir_reg)				\
 	sldi	stack_reg,pir_reg,STACK_SHIFT;			\
 	addis	stack_reg,stack_reg,CPU_STACKS_OFFSET@ha;	\
@@ -471,27 +473,7 @@ call_relocate:
        .long 0xa6037b7d; /* mtsrr1 r11                         */ \
        .long 0x2400004c  /* rfid                               */
 
-.global enter_pm_state
-enter_pm_state:
-	/* Before entering map or rvwinkle, we create a stack frame
-	 * and save our non-volatile registers.
-	 *
-	 * We also save these SPRs:
-	 *
-	 *  - HSPRG0	in GPR0 slot
-	 *  - HSPRG1	in GPR1 slot
-	 *
-	 *  - xxx TODO: HIDs
-	 *  - TODO: Mask MSR:ME during the process
-	 *
-	 * On entry, r3 indicates:
-	 *
-	 *    0 = nap
-	 *    1 = rvwinkle
-	 */
-	mflr	%r0
-	std	%r0,16(%r1)
-	stdu	%r1,-STACK_FRAMESIZE(%r1)
+pm_save_regs:
 	SAVE_GPR(2,%r1)
 	SAVE_GPR(14,%r1)
 	SAVE_GPR(15,%r1)
@@ -519,6 +501,31 @@ enter_pm_state:
 	stw	%r5,STACK_XER(%r1)
 	std	%r6,STACK_GPR0(%r1)
 	std	%r7,STACK_GPR1(%r1)
+	blr
+
+.global enter_p8_pm_state
+enter_p8_pm_state:
+	/* Before entering map or rvwinkle, we create a stack frame
+	 * and save our non-volatile registers.
+	 *
+	 * We also save these SPRs:
+	 *
+	 *  - HSPRG0	in GPR0 slot
+	 *  - HSPRG1	in GPR1 slot
+	 *
+	 *  - xxx TODO: HIDs
+	 *  - TODO: Mask MSR:ME during the process
+	 *
+	 * On entry, r3 indicates:
+	 *
+	 *    0 = nap
+	 *    1 = rvwinkle
+	 */
+	mflr	%r0
+	std	%r0,16(%r1)
+	stdu	%r1,-STACK_FRAMESIZE(%r1)
+
+	bl	pm_save_regs
 
 	/* Save stack pointer in struct cpu_thread */
 	std	%r1,CPUTHREAD_SAVE_R1(%r13)
@@ -543,6 +550,27 @@ enter_pm_state:
 	PPC_INST_RVWINKLE
 	b	.
 
+.global enter_p9_pm_lite_state
+enter_p9_pm_lite_state:
+	mtspr	SPR_PSSCR,%r3
+	PPC_INST_STOP
+	blr
+
+.global enter_p9_pm_state
+enter_p9_pm_state:
+	mflr	%r0
+	std	%r0,16(%r1)
+	stdu	%r1,-STACK_FRAMESIZE(%r1)
+
+	bl	pm_save_regs
+
+	/* Save stack pointer in struct cpu_thread */
+	std	%r1,CPUTHREAD_SAVE_R1(%r13)
+
+	mtspr	SPR_PSSCR,%r3
+	PPC_INST_STOP
+	b	.
+
 /* This is a little piece of code that is copied down to
  * 0x100 for handling power management wakeups
  */

core/cpu.c

@@ -94,8 +94,12 @@ static void cpu_wake(struct cpu_thread *cpu)
 	if (!cpu->in_idle)
 		return;
 
-	/* Poke IPI */
-	icp_kick_cpu(cpu);
+	if (proc_gen == proc_gen_p8 || proc_gen == proc_gen_p7) {
+		/* Poke IPI */
+		icp_kick_cpu(cpu);
+	} else if (proc_gen == proc_gen_p9) {
+		p9_dbell_send(cpu->pir);
+	}
 }
 
 static struct cpu_thread *cpu_find_job_target(void)
@@ -319,24 +323,28 @@ static void cpu_idle_p8(enum cpu_wake_cause wake_on)
 		if (cpu_check_jobs(cpu) || !pm_enabled)
 			goto skip_sleep;
 
+		/* Setup wakup cause in LPCR: EE (for IPI) */
 		lpcr |= SPR_LPCR_P8_PECE2;
 		mtspr(SPR_LPCR, lpcr);
 
 	} else {
-		/* Mark outselves sleeping so wakeup knows to send an IPI */
+		/* Mark outselves sleeping so cpu_set_pm_enable knows to
+		 * send an IPI
+		 */
 		cpu->in_sleep = true;
 		sync();
 
 		/* Check if PM got disabled */
 		if (!pm_enabled)
 			goto skip_sleep;
 
+		/* EE and DEC */
 		lpcr |= SPR_LPCR_P8_PECE2 | SPR_LPCR_P8_PECE3;
 		mtspr(SPR_LPCR, lpcr);
 	}
 
 	/* Enter nap */
-	enter_pm_state(false);
+	enter_p8_pm_state(false);
 
 skip_sleep:
 	/* Restore */
@@ -346,12 +354,78 @@ static void cpu_idle_p8(enum cpu_wake_cause wake_on)
 	reset_cpu_icp();
 }
 
+static void cpu_idle_p9(enum cpu_wake_cause wake_on)
+{
+	uint64_t lpcr = mfspr(SPR_LPCR) & ~SPR_LPCR_P9_PECE;
+	uint64_t psscr;
+	struct cpu_thread *cpu = this_cpu();
+
+	if (!pm_enabled) {
+		prlog_once(PR_DEBUG, "cpu_idle_p9 called pm disabled\n");
+		return;
+	}
+
+	msgclr(); /* flush pending messages */
+
+	/* Synchronize with wakers */
+	if (wake_on == cpu_wake_on_job) {
+		/* Mark ourselves in idle so other CPUs know to send an IPI */
+		cpu->in_idle = true;
+		sync();
+
+		/* Check for jobs again */
+		if (cpu_check_jobs(cpu) || !pm_enabled)
+			goto skip_sleep;
+
+		/* HV DBELL for IPI */
+		lpcr |= SPR_LPCR_P9_PECEL1;
+	} else {
+		/* Mark outselves sleeping so cpu_set_pm_enable knows to
+		 * send an IPI
+		 */
+		cpu->in_sleep = true;
+		sync();
+
+		/* Check if PM got disabled */
+		if (!pm_enabled)
+			goto skip_sleep;
+
+		/* HV DBELL and DEC */
+		lpcr |= SPR_LPCR_P9_PECEL1 | SPR_LPCR_P9_PECEL3;
+		mtspr(SPR_LPCR, lpcr);
+	}
+
+	mtspr(SPR_LPCR, lpcr);
+
+	if (sreset_enabled) {
+		/* stop with EC=1 (sreset) and ESL=1 (enable thread switch). */
+		/* PSSCR SD=0 ESL=1 EC=1 PSSL=0 TR=3 MTL=0 RL=3 */
+		psscr = PPC_BIT(42) | PPC_BIT(43) |
+			PPC_BITMASK(54, 55) | PPC_BITMASK(62,63);
+		enter_p9_pm_state(psscr);
+	} else {
+		/* stop with EC=0 (resumes) which does not require sreset. */
+		/* PSSCR SD=0 ESL=0 EC=0 PSSL=0 TR=3 MTL=0 RL=3 */
+		psscr = PPC_BITMASK(54, 55) | PPC_BITMASK(62,63);
+		enter_p9_pm_lite_state(psscr);
+	}
+
+skip_sleep:
+	/* Restore */
+	cpu->in_idle = false;
+	cpu->in_sleep = false;
+	p9_dbell_receive();
+}
+
 static void cpu_idle_pm(enum cpu_wake_cause wake_on)
 {
 	switch(proc_gen) {
 	case proc_gen_p8:
 		cpu_idle_p8(wake_on);
 		break;
+	case proc_gen_p9:
+		cpu_idle_p9(wake_on);
+		break;
 	default:
 		prlog_once(PR_DEBUG, "cpu_idle_pm called with bad processor type\n");
 		break;
@@ -429,6 +503,18 @@ static void cpu_pm_disable(void)
 				cpu_relax();
 			}
 		}
+	} else if (proc_gen == proc_gen_p9) {
+		for_each_available_cpu(cpu) {
+			if (cpu->in_sleep || cpu->in_idle)
+				p9_dbell_send(cpu->pir);
+		}
+
+		smt_lowest();
+		for_each_available_cpu(cpu) {
+			while (cpu->in_sleep || cpu->in_idle)
+				barrier();
+		}
+		smt_medium();
 	}
 }
 
@@ -451,6 +537,22 @@ void cpu_set_sreset_enable(bool enabled)
 			if (ipi_enabled)
 				pm_enabled = true;
 		}
+
+	} else if (proc_gen == proc_gen_p9) {
+		/* Don't use sreset idle on DD1 (has a number of bugs) */
+		uint32_t version = mfspr(SPR_PVR);
+		if (is_power9n(version) && (PVR_VERS_MAJ(version) == 1))
+			return;
+
+		sreset_enabled = enabled;
+		sync();
+		/*
+		 * Kick everybody out of PM so they can adjust the PM
+		 * mode they are using (EC=0/1).
+		 */
+		cpu_pm_disable();
+		if (ipi_enabled)
+			pm_enabled = true;
 	}
 }
 
@@ -468,6 +570,19 @@ void cpu_set_ipi_enable(bool enabled)
 			if (sreset_enabled)
 				pm_enabled = true;
 		}
+
+	} else if (proc_gen == proc_gen_p9) {
+		/* Don't use doorbell on DD1 (requires darn for msgsync) */
+		uint32_t version = mfspr(SPR_PVR);
+		if (is_power9n(version) && (PVR_VERS_MAJ(version) == 1))
+			return;
+
+		ipi_enabled = enabled;
+		sync();
+		if (!enabled)
+			cpu_pm_disable();
+		else
+			pm_enabled = true;
 	}
 }
 

core/init.c

@@ -928,6 +928,8 @@ void __noreturn __nomcount main_cpu_entry(const void *fdt)
 
 	/* Initialize the rest of the cpu thread structs */
 	init_all_cpus();
+	if (proc_gen == proc_gen_p9)
+		cpu_set_ipi_enable(true);
 
 	/* Allocate our split trace buffers now. Depends add_opal_node() */
 	init_trace_buffers();

hw/slw.c

@@ -83,7 +83,7 @@ static void slw_do_rvwinkle(void *data)
 	/* Tell that we got it */
 	cpu->state = cpu_state_rvwinkle;
 
-	enter_pm_state(1);
+	enter_p8_pm_state(1);
 
 	/* Restore SPRs */
 	init_shared_sprs();

include/processor.h

@@ -77,6 +77,7 @@
 #define SPR_HMER	0x150	/* Hypervisor Maintenance Exception */
 #define SPR_HMEER	0x151	/* HMER interrupt enable mask */
 #define SPR_AMOR	0x15d
+#define SPR_PSSCR	0x357   /* RW: Stop status and control (ISA 3) */
 #define SPR_TSCR	0x399
 #define SPR_HID0	0x3f0
 #define SPR_HID1	0x3f1
@@ -85,6 +86,7 @@
 #define SPR_HID5	0x3f6
 #define SPR_PIR		0x3ff	/* RO: Processor Identification */
 
+
 /* Bits in LPCR */
 
 /* Powersave Exit Cause Enable is different for P7 and P8 */
@@ -99,6 +101,14 @@
 #define SPR_LPCR_P8_PECE2	PPC_BIT(49)   /* Wake on external interrupts */
 #define SPR_LPCR_P8_PECE3	PPC_BIT(50)   /* Wake on decrementer */
 #define SPR_LPCR_P8_PECE4	PPC_BIT(51)   /* Wake on MCs, HMIs, etc... */
+
+#define SPR_LPCR_P9_PECE	(PPC_BITMASK(47,51) | PPC_BITMASK(17,17))
+#define SPR_LPCR_P9_PECEU0	PPC_BIT(17)   /* Wake on HVI */
+#define SPR_LPCR_P9_PECEL0	PPC_BIT(47)   /* Wake on priv doorbell */
+#define SPR_LPCR_P9_PECEL1	PPC_BIT(48)   /* Wake on hv doorbell */
+#define SPR_LPCR_P9_PECEL2	PPC_BIT(49)   /* Wake on external interrupts */
+#define SPR_LPCR_P9_PECEL3	PPC_BIT(50)   /* Wake on decrementer */
+#define SPR_LPCR_P9_PECEL4	PPC_BIT(51)   /* Wake on MCs, HMIs, etc... */
 #define SPR_LPCR_P9_LD		PPC_BIT(46)   /* Large decrementer mode bit */
 
 
@@ -309,6 +319,27 @@ static inline void sync_icache(void)
 	asm volatile("sync; icbi 0,%0; sync; isync" : : "r" (0) : "memory");
 }
 
+/*
+ * Doorbells
+ */
+static inline void msgclr(void)
+{
+	uint64_t rb = (0x05 << (63-36));
+	asm volatile("msgclr %0" : : "r"(rb));
+}
+
+static inline void p9_dbell_receive(void)
+{
+	uint64_t rb = (0x05 << (63-36));
+	/* msgclr ; msgsync ; lwsync */
+	asm volatile("msgclr %0 ; .long 0x7c0006ec ; lwsync" : : "r"(rb));
+}
+
+static inline void p9_dbell_send(uint32_t pir)
+{
+	uint64_t rb = (0x05 << (63-36)) | pir;
+	asm volatile("sync ; msgsnd %0" : : "r"(rb));
+}
 
 /*
  * Byteswap load/stores

include/skiboot.h

@@ -315,7 +315,9 @@ extern void fast_sleep_exit(void);
 extern void fake_rtc_init(void);
 
 /* Assembly in head.S */
-extern void enter_pm_state(bool winkle);
+extern void enter_p8_pm_state(bool winkle);
+extern void enter_p9_pm_state(uint64_t psscr);
+extern void enter_p9_pm_lite_state(uint64_t psscr);
 extern uint32_t reset_patch_start;
 extern uint32_t reset_patch_end;