UCT/IB: Support VRF tables for RoCE reachability check

Signed-off-by: Dmitrii Gabor <dmitryg1709@gmail.com>

Author: dmitrygx

AUTHORS

@@ -23,6 +23,7 @@ Corey J. Nolet <cjnolet@gmail.com>
 David Wootton <dwootton@us.ibm.com>
 Devendar Bureddy <devendar@mellanox.com>
 Devesh Sharma <devesh.sharma@broadcom.com>
+Dmitrii Gabor <dmitryg1709@gmail.com>
 Dmitry Gladkov <dmitrygla@mellanox.com>
 Doug Jacobsen <dmjacobsen@lbl.gov>
 Edgar Gabriel <edgar.gabriel@amd.com>

src/ucs/sys/sys.c

@@ -165,6 +165,18 @@ uint32_t ucs_file_checksum(const char *filename)
     return crc;
 }
 
+ucs_status_t ucs_ifname_to_index(const char *ndev_name, unsigned *ndev_index_p)
+{
+    unsigned ndev_index = if_nametoindex(ndev_name);
+    if (ndev_index == 0) {
+        ucs_error("failed to get interface index for %s: %m", ndev_name);
+        return UCS_ERR_IO_ERROR;
+    }
+
+    *ndev_index_p = ndev_index;
+    return UCS_OK;
+}
+
 static uint64_t ucs_get_mac_address()
 {
     static uint64_t mac_address = 0;

src/ucs/sys/sys.h

@@ -186,6 +186,12 @@ const char *ucs_get_exe();
 uint32_t ucs_file_checksum(const char *filename);
 
 
+/**
+ * Get interface index for a given interface name.
+ */
+ucs_status_t ucs_ifname_to_index(const char *ndev_name, unsigned *ndev_index_p);
+
+
 /**
  * Get a globally unique identifier of the machine running the current process.
  */

src/uct/ib/base/ib_device.c

@@ -30,6 +30,8 @@
 #include <rdma/rdma_netlink.h>
 #endif
 
+#define UCT_IB_DEVICE_LOOPBACK_NDEV_INDEX_INVALID 0
+
 
 /* This table is according to "Encoding for RNR NAK Timer Field"
  * in IBTA specification */
@@ -109,7 +111,7 @@ uct_ib_device_to_ndev_cache_hash_equal(uct_ib_device_to_ndev_key_t key1,
            (key1.guid == key2.guid);
 }
 
-KHASH_INIT(uct_ib_device_to_ndev, uct_ib_device_to_ndev_key_t, int, 1,
+KHASH_INIT(uct_ib_device_to_ndev, uct_ib_device_to_ndev_key_t, unsigned, 1,
            uct_ib_device_to_ndev_cache_hash_func,
            uct_ib_device_to_ndev_cache_hash_equal);
 
@@ -1510,9 +1512,25 @@ uct_ib_device_get_roce_ndev_name(uct_ib_device_t *dev, uint8_t port_num,
     return UCS_OK;
 }
 
+ucs_status_t uct_ib_iface_get_loopback_ndev_index(unsigned *ndev_index_p)
+{
+    static unsigned loopback_ndev_index = UCT_IB_DEVICE_LOOPBACK_NDEV_INDEX_INVALID;
+    ucs_status_t status;
+
+    if (loopback_ndev_index == UCT_IB_DEVICE_LOOPBACK_NDEV_INDEX_INVALID) {
+        status = ucs_ifname_to_index("lo", &loopback_ndev_index);
+        if (status != UCS_OK) {
+            return status;
+        }
+    }
+
+    *ndev_index_p = loopback_ndev_index;
+    return UCS_OK;
+}
+
 ucs_status_t
 uct_ib_device_get_roce_ndev_index(uct_ib_device_t *dev, uint8_t port_num,
-                                  uint8_t gid_index, int *ndev_index_p)
+                                  uint8_t gid_index, unsigned *ndev_index_p)
 {
     uct_ib_device_to_ndev_key_t ib_dev = {.guid = IBV_DEV_ATTR(dev, node_guid),
                                           .port_num = port_num,
@@ -1521,9 +1539,9 @@ uct_ib_device_get_roce_ndev_index(uct_ib_device_t *dev, uint8_t port_num,
                                           PTHREAD_MUTEX_INITIALIZER;
     ucs_status_t status;
     char ndev_name[IFNAMSIZ];
-    int ndev_index;
+    unsigned ndev_index;
     khiter_t iter;
-    int khret;
+    unsigned khret;
 
     pthread_mutex_lock(&uct_ib_device_to_ndev_cache_lock);
     iter = kh_put(uct_ib_device_to_ndev, &ib_dev_to_ndev_map, ib_dev, &khret);
@@ -1539,11 +1557,8 @@ uct_ib_device_get_roce_ndev_index(uct_ib_device_t *dev, uint8_t port_num,
             goto out_unlock;
         }
 
-        ndev_index = if_nametoindex(ndev_name);
-        if (ndev_index == 0) {
-            ucs_error("failed to get interface index for %s (errno %d)",
-                      ndev_name, errno);
-            status = UCS_ERR_IO_ERROR;
+        status = ucs_ifname_to_index(ndev_name, &ndev_index);
+        if (status != UCS_OK) {
             goto out_unlock;
         }
 

src/uct/ib/base/ib_device.h

@@ -409,9 +409,11 @@ ucs_status_t uct_ib_device_get_roce_ndev_name(uct_ib_device_t *dev,
                                               uint8_t gid_index,
                                               char *ndev_name, size_t max);
 
+ucs_status_t uct_ib_iface_get_loopback_ndev_index(unsigned *ndev_index_p);
+
 ucs_status_t
 uct_ib_device_get_roce_ndev_index(uct_ib_device_t *dev, uint8_t port_num,
-                                  uint8_t gid_index, int *ndev_index_p);
+                                  uint8_t gid_index, unsigned *ndev_index_p);
 
 unsigned uct_ib_device_get_roce_lag_level(uct_ib_device_t *dev,
                                           uint8_t port_num,

src/uct/ib/base/ib_iface.c

@@ -692,18 +692,40 @@ uct_ib_iface_roce_is_routable(uct_ib_iface_t *iface, uint8_t gid_index,
     uct_ib_device_t *dev = uct_ib_iface_device(iface);
     uint8_t port_num     = iface->config.port_num;
     char remote_str[128];
-    int ndev_index;
+    unsigned ndev_index, lo_ndev_index;
 
     if (uct_ib_device_get_roce_ndev_index(dev, port_num, gid_index,
                                           &ndev_index) != UCS_OK) {
-        uct_iface_fill_info_str_buf(params, "iface index is not found");
+        uct_iface_fill_info_str_buf(params,
+                                    "iface index is not found for "
+                                    UCT_IB_IFACE_FMT ", gid index %u",
+                                    UCT_IB_IFACE_ARG(iface), gid_index);
         return 0;
     }
 
     if (!ucs_netlink_route_exists(ndev_index, sa_remote)) {
-        uct_iface_fill_info_str_buf(params, "remote address %s is not routable",
-                                    ucs_sockaddr_str(sa_remote, remote_str, 128));
-        return 0;
+        /* try to use loopback interface for reachability check, because it may
+         * be used for routing in case of an interface with VRF is configured
+         * and a RoCE IP interface uses this VRF table for routing.
+         */
+        if (uct_ib_iface_get_loopback_ndev_index(&lo_ndev_index) != UCS_OK) {
+            uct_iface_fill_info_str_buf(params,
+                                        "loopback iface index is not found");
+            return 0;
+        }
+
+        if (!ucs_netlink_route_exists(lo_ndev_index, sa_remote)) {
+            uct_iface_fill_info_str_buf(params,
+                                        "remote address %s is not routable "
+                                        "neither by interface "UCT_IB_IFACE_FMT
+                                        " (ifname_index=%u) nor by loopback "
+                                        "interface (ifname_index=%u)",
+                                        ucs_sockaddr_str(sa_remote, remote_str,
+                                                         128),
+                                        UCT_IB_IFACE_ARG(iface),
+                                        ndev_index, lo_ndev_index);
+            return 0;
+        }
     }
 
     return 1;