xive: disable store EOI support

Hardware has limitations which would require to put a sync after each
store EOI to make sure the MMIO operations that change the ESB state
are ordered. This is a killer for performance and the PHBs do not
support the sync. So remove the store EOI for the moment, until
hardware is improved.

Also, while we are at changing the XIVE source flags, let's fix the
settings for the PHB4s which should follow these rules :

  - SHIFT_BUG    for DD10
  - STORE_EOI    for DD20 and if enabled
  - TRIGGER_PAGE for DDx0 and if not STORE_EOI

Signed-off-by: Cédric Le Goater <clg@kaod.org>
Signed-off-by: Stewart Smith <stewart@linux.vnet.ibm.com>

hw/phb4.c

@@ -145,6 +145,9 @@ static void phb4_init_hw(struct phb4 *p, bool first_init);
 #define PHBLOGCFG(p, fmt, a...) do {} while (0)
 #endif
 
+#define PHB4_CAN_STORE_EOI(p) \
+	(XIVE_STORE_EOI_ENABLED && ((p)->rev >= PHB4_REV_NIMBUS_DD20))
+
 static bool verbose_eeh;
 static bool pci_tracing;
 static bool pci_eeh_mmio;
@@ -4660,7 +4663,8 @@ static void phb4_init_hw(struct phb4 *p, bool first_init)
 		val |= SETFIELD(PHB_CTRLR_TVT_ADDR_SEL, 0ull, TVT_DD1_2_PER_PE);
 	} else {
 		val |= SETFIELD(PHB_CTRLR_TVT_ADDR_SEL, 0ull, TVT_2_PER_PE);
-		val |= PHB_CTRLR_IRQ_STORE_EOI;
+		if (PHB4_CAN_STORE_EOI(p))
+			val |= PHB_CTRLR_IRQ_STORE_EOI;
 	}
 
 	if (!pci_eeh_mmio)
@@ -5255,10 +5259,16 @@ static void phb4_create(struct dt_node *np)
 	/* Load capp microcode into capp unit */
 	load_capp_ucode(p);
 
-	/* Register all interrupt sources with XIVE */
-	irq_flags = XIVE_SRC_SHIFT_BUG | XIVE_SRC_TRIGGER_PAGE;
-	if (p->rev >= PHB4_REV_NIMBUS_DD20)
+	/* Compute XIVE source flags depending on PHB revision */
+	irq_flags = 0;
+	if (p->rev == PHB4_REV_NIMBUS_DD10)
+		irq_flags |= XIVE_SRC_SHIFT_BUG;
+	if (PHB4_CAN_STORE_EOI(p))
 		irq_flags |= XIVE_SRC_STORE_EOI;
+	else
+		irq_flags |= XIVE_SRC_TRIGGER_PAGE;
+
+	/* Register all interrupt sources with XIVE */
 	xive_register_hw_source(p->base_msi, p->num_irqs - 8, 16,
 				p->int_mmio, irq_flags, NULL, NULL);
 

hw/xive.c

@@ -487,6 +487,9 @@ struct xive {
 	void		*q_ovf;
 };
 
+#define XIVE_CAN_STORE_EOI(x) \
+	(XIVE_STORE_EOI_ENABLED && ((x)->rev >= XIVE_REV_2))
+
 /* Global DT node */
 static struct dt_node *xive_dt_node;
 
@@ -1751,7 +1754,11 @@ static bool xive_config_init(struct xive *x)
 
 	/* Enable StoreEOI */
 	val = xive_regr(x, VC_SBC_CONFIG);
-	val |= VC_SBC_CONF_CPLX_CIST | VC_SBC_CONF_CIST_BOTH;
+	if (XIVE_CAN_STORE_EOI(x))
+		val |= VC_SBC_CONF_CPLX_CIST | VC_SBC_CONF_CIST_BOTH;
+	else
+		xive_dbg(x, "store EOI is disabled\n");
+
 	val |= VC_SBC_CONF_NO_UPD_PRF;
 	xive_regw(x, VC_SBC_CONFIG, val);
 
@@ -2797,7 +2804,7 @@ void xive_register_ipi_source(uint32_t base, uint32_t count, void *data,
 	assert(s);
 
 	/* Store EOI supported on DD2.0 */
-	if (x->rev >= XIVE_REV_2)
+	if (XIVE_CAN_STORE_EOI(x))
 		flags |= XIVE_SRC_STORE_EOI;
 
 	/* Callbacks assume the MMIO base corresponds to the first
@@ -2905,7 +2912,7 @@ static struct xive *init_one_xive(struct dt_node *np)
 
 	/* Register built-in source controllers (aka IPIs) */
 	flags = XIVE_SRC_EOI_PAGE1 | XIVE_SRC_TRIGGER_PAGE;
-	if (x->rev >= XIVE_REV_2)
+	if (XIVE_CAN_STORE_EOI(x))
 		flags |= XIVE_SRC_STORE_EOI;
 	__xive_register_source(x, &x->ipis, x->int_base,
 			       x->int_hw_bot - x->int_base, IPI_ESB_SHIFT,

include/xive.h

@@ -492,6 +492,9 @@ uint32_t xive_alloc_ipi_irqs(uint32_t chip_id, uint32_t count, uint32_t align);
 uint64_t xive_get_notify_port(uint32_t chip_id, uint32_t ent);
 uint32_t xive_get_notify_base(uint32_t girq);
 
+/* XIVE feature flag to de/activate store EOI */
+#define XIVE_STORE_EOI_ENABLED 0
+
 /* Internal IRQ flags */
 #define XIVE_SRC_TRIGGER_PAGE	0x01 /* Trigger page exist (either separate
 				      * or not, so different from the OPAL