npu3: Register virtual PHBs with static IDs

Assigning opal IDs to virtual PHBs dynamically may lead to userland
seeing the PCI domain ID for an adapter vary when adding or removing
another adapter (GPU or opencapi).

This patch switches to using static opal IDs for virtual PHBs, based
on their ibm,phb-index property, which was made static by a previous
patch.

Note that the PCI domain IDs will increase on the second chip (or
more, if we had more) because we now reserve 16 IDs per chip for PHBs.

This affects Axone only. We don't change anything on P9 and npu2, to
avoid altering how domain IDs have been shown on already GA'd
platforms.

Reviewed-by: Reza Arbab <arbab@linux.ibm.com>
Reviewed-by: Andrew Donnellan <ajd@linux.ibm.com>
Signed-off-by: Frederic Barrat <fbarrat@linux.ibm.com>
Signed-off-by: Oliver O'Halloran <oohall@gmail.com>

hw/npu3-nvlink.c

@@ -904,7 +904,8 @@ static void npu3_create_phb(struct npu3 *npu)
 	assert(phb->dt_node);
 
 	list_head_init(&phb->virt_devices);
-	pci_register_phb(phb, OPAL_DYNAMIC_PHB_ID);
+	pci_register_phb(phb, npu3_get_opal_id(npu->chip_id,
+					       npu3_get_phb_index(npu->index)));
 	npu3_create_phb_slot(npu);
 	npu3_ioda_reset(phb, true);
 }

include/npu3.h

@@ -20,6 +20,7 @@
 #include <phys-map.h>
 #include <pci.h>
 #include <npu3-regs.h>
+#include <phb4.h>
 
 enum npu3_dev_type {
 	NPU3_DEV_TYPE_UNKNOWN = 0,
@@ -183,4 +184,9 @@ static inline int npu3_get_phb_index(unsigned int npu_index)
 	return NPU3_PHB_INDEX_BASE + npu_index;
 }
 
+static inline int npu3_get_opal_id(unsigned int chip_id, unsigned int index)
+{
+	return phb4_get_opal_id(chip_id, index);
+}
+
 #endif /* __NPU3_H */

include/phb4.h

@@ -246,10 +246,15 @@ static inline void phb4_set_err_pending(struct phb4 *p, bool pending)
 }
 
 #define PHB4_PER_CHIP                        6 /* Max 6 PHBs per chip on p9 */
+#define PHB4_MAX_PHBS_PER_CHIP_P9            PHB4_PER_CHIP
+#define PHB4_MAX_PHBS_PER_CHIP_P9P           0x10 /* extra for virt PHBs */
 
 static inline int phb4_get_opal_id(unsigned int chip_id, unsigned int index)
 {
-	return chip_id * PHB4_PER_CHIP + index;
+	if (PVR_TYPE(mfspr(SPR_PVR)) == PVR_TYPE_P9)
+		return chip_id * PHB4_MAX_PHBS_PER_CHIP_P9 + index;
+	else
+		return chip_id * PHB4_MAX_PHBS_PER_CHIP_P9P + index;
 }
 
 #endif /* __PHB4_H */