NPU2: Add flag to nvlink config space indicating DL reset state

Device drivers need to be able to determine if the DL is out of reset or
not so they can safely probe to see if links have already been trained.
This patch adds a flag to the vendor specific config space indicating if
the DL is out of reset.

Signed-off-by: Alistair Popple <alistair@popple.id.au>
Signed-off-by: Stewart Smith <stewart@linux.vnet.ibm.com>

doc/nvlink.rst

@@ -146,8 +146,8 @@ Procedure Status Register
 
 PCI Device Flag
 
-   Bit 0 is set only if an actual PCI device was bound to this
-   emulated device.
+   Bit 0 - set if the GPU PCIe device associated with this nvlink was found.
+   bit 1 - set if the DL has been taken out of reset.
 
 Link Number
 

hw/npu2-hw-procedures.c

@@ -241,6 +241,8 @@ static uint32_t reset_ntl_release(struct npu2_dev *ndev)
 	npu2_write(ndev->npu, NPU2_NTL_CRED_DATA_CREDIT_RX(ndev), 0x0001000000000000);
 	npu2_write(ndev->npu, NPU2_NTL_RSP_DATA_CREDIT_RX(ndev), 0x0001000000000000);
 
+	npu2_set_link_flag(ndev, NPU2_DEV_DL_RESET);
+
 	return PROCEDURE_COMPLETE;
 }
 DEFINE_PROCEDURE(reset_ntl, reset_ndl, reset_ntl_release);

hw/npu2.c

@@ -121,6 +121,14 @@ void npu2_write_mask(struct npu2 *p, uint64_t reg, uint64_t val, uint64_t mask)
 	npu2_scom_write(p->chip_id, p->xscom_base, reg, NPU2_MISC_DA_LEN_8B, new_val);
 }
 
+/* Set a specific flag in the vendor config space */
+void npu2_set_link_flag(struct npu2_dev *ndev, uint8_t flag)
+{
+	ndev->link_flags |= flag;
+	PCI_VIRT_CFG_INIT_RO(ndev->pvd, VENDOR_CAP_START +
+			     VENDOR_CAP_PCI_DEV_OFFSET, 1, ndev->link_flags);
+}
+
 static inline void npu2_ioda_sel(struct npu2 *p, uint32_t table,
 				uint32_t index, bool autoinc)
 {
@@ -442,8 +450,7 @@ static void npu2_dev_bind_pci_dev(struct npu2_dev *dev)
 		if (dev->pd) {
 			dev->phb = phb;
 			/* Found the device, set the bit in config space */
-			PCI_VIRT_CFG_INIT_RO(dev->pvd, VENDOR_CAP_START +
-				VENDOR_CAP_PCI_DEV_OFFSET, 1, 0x01);
+			npu2_set_link_flag(dev, NPU2_DEV_PCI_LINKED);
 			return;
 		}
 	}

include/npu2.h

@@ -35,6 +35,10 @@
 
 #define NPU2_LINKS_PER_CHIP 6
 
+/* Link flags */
+#define NPU2_DEV_PCI_LINKED	0x1
+#define NPU2_DEV_DL_RESET	0x2
+
 /* Return the stack (0-2) of a device */
 #define NPU2DEV_STACK(ndev) ((ndev)->index / 2)
 
@@ -89,7 +93,7 @@ struct npu2_dev {
 	struct phb		*phb;
 	struct pci_device	*pd;
 
-	int                     ntl_reset_done;
+	uint8_t			link_flags;
 
 	/* Vendor specific capability */
 	uint32_t		vendor_cap;
@@ -147,5 +151,6 @@ void npu2_write_mask(struct npu2 *p, uint64_t reg, uint64_t val, uint64_t mask);
 int64_t npu2_dev_procedure(void *dev, struct pci_cfg_reg_filter *pcrf,
 			   uint32_t offset, uint32_t len, uint32_t *data,
 			   bool write);
+void npu2_set_link_flag(struct npu2_dev *ndev, uint8_t flag);
 extern int nv_zcal_nominal;
 #endif /* __NPU2_H */