Check rshim accessibility when re-enabling it

This commit checks the rshim accessibility when re-enabling it.
It fixes an issue in the following scenario:

1. rshim_pcie or rshim_pcie_lf is running;
2. MFT tool sets the rshim DROP_MODE to 1 and starts FW reset;
3. FW reset triggers the BlueField reset and thus USB reset;
4. USB rshim driver running on another host takes over this rshim
   device (because the local rshim driver is still in DROP_MODE
   which prevents the scratchpad register access. Thus the USB
   rshim driver assumes no one is using the rshim device).
5. After FW reset, the rshim_pcie/rshim_pcie_lf driver resumes the
   rshim access by setting DROP_MODE to 0.
6. Now there are two rshim drivers accessing the same rshim device
   which caused unexpected behavior.

The fix is to check rshim accessibility at step 5 above to make
sure no one is using it when re-enabling it.

Signed-off-by: Liming Sun <lsun@mellanox.com>

src/rshim.c

@@ -2107,8 +2107,7 @@ static int rshim_bf2_a0_wa(rshim_backend_t *bd)
   return 0;
 }
 
-/* Check whether backend is allowed to register or not. */
-static int rshim_access_check(rshim_backend_t *bd)
+int rshim_access_check(rshim_backend_t *bd)
 {
   rshim_backend_t *other_bd;
   uint64_t value;

src/rshim.h

@@ -546,4 +546,7 @@ static inline bool rshim_drop_mode_access(int addr)
 int rshim_get_opn(rshim_backend_t *bd, char *opn, int len);
 int rshim_set_opn(rshim_backend_t *bd, const char *opn, int len);
 
+/* Check whether rshim backend is accessible or not. */
+int rshim_access_check(rshim_backend_t *bd);
+
 #endif /* _RSHIM_H */

src/rshim_fuse.c

@@ -786,7 +786,7 @@ static int rshim_fuse_misc_write(struct cuse_dev *cdev, int fflags,
   rshim_backend_t *bd = cuse_dev_get_priv0(cdev);
   const char *p = buf;
 #endif
-  int i, rc = 0, value = 0, mac[6], vlan[2] = {0};
+  int i, rc = 0, value = 0, mac[6], vlan[2] = {0}, old_value;
   char opn[RSHIM_YU_BOOT_RECORD_OPN_SIZE + 1] = "";
   char key[32];
 
@@ -828,13 +828,26 @@ static int rshim_fuse_misc_write(struct cuse_dev *cdev, int fflags,
   } else if (strcmp(key, "DROP_MODE") == 0) {
     if (sscanf(p, "%d", &value) != 1)
       goto invalid;
+    old_value = (int)bd->drop_mode;
     bd->drop_mode = !!value;
     if (bd->drop_mode)
       bd->drop_pkt = 1;
     if (bd->enable_device) {
       if (bd->enable_device(bd, true))
         bd->drop_mode = 1;
     }
+    /*
+     * Check if another endpoint driver has already attached to the
+     * same rshim device before enabling it.
+     */
+    if (old_value && !bd->drop_mode) {
+      pthread_mutex_lock(&bd->mutex);
+      if (rshim_access_check(bd)) {
+        RSHIM_WARN("rshim %s is not accessible\n", bd->dev_name);
+        bd->drop_mode = old_value;
+      }
+      pthread_mutex_unlock(&bd->mutex);
+    }
   } else if (strcmp(key, "BOOT_MODE") == 0) {
     if (sscanf(p, "%x", &value) != 1)
       goto invalid;

src/rshim_pcie.c

@@ -500,6 +500,7 @@ int rshim_pcie_enable(void *dev, bool enable)
            pci_dev->dev, pci_dev->func);
   dir = opendir(path);
   if (dir) {
+    RSHIM_ERR("failed to open %s\n", path);
     closedir(dir);
     return -EBUSY;
   }
@@ -509,19 +510,22 @@ int rshim_pcie_enable(void *dev, bool enable)
            SYS_BUS_PCI, pci_dev->domain, pci_dev->bus,
            pci_dev->dev, pci_dev->func);
   fd = open(path, O_RDWR | O_CLOEXEC);
-  if (fd == -1)
+  if (fd == -1) {
+    RSHIM_ERR("failed to open %s\n", path);
     return -errno;
+  }
   if (write(fd, enable ? "1" : "0", 1) < 0)
     rc = -errno;
   close(fd);
-  if (rc)
+  if (rc) {
+    RSHIM_ERR("failed to write to %s\n", path);
     return rc;
+  }
 
   pci_write_word(pci_dev, PCI_COMMAND, PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER);
-  return rc;
-#else
-  return 0;
 #endif
+
+  return 0;
 }
 
 int rshim_pcie_init(void)