Merge pull request #946 from philip-davis/GetPriority

Assign conditional priority value to RDMA DP
ornladios · Oct 30, 2018 · 6b66ce7 · 6b66ce7
2 parents 46060ef + 8c14133
commit 6b66ce7
Showing 1 changed file with 95 additions and 14 deletions.
diff --git a/source/adios2/toolkit/sst/dp/rdma_dp.c b/source/adios2/toolkit/sst/dp/rdma_dp.c
@@ -89,7 +89,7 @@ struct fabric_state
 
 static void init_fabric(struct fabric_state *fabric)
 {
-    struct fi_info *hints, *info;
+    struct fi_info *hints, *info, *originfo, *useinfo;
     struct fi_av_attr av_attr = {0};
     struct fi_cq_attr cq_attr = {0};
     char *ifname;
@@ -106,11 +106,37 @@ static void init_fabric(struct fabric_state *fabric)
     fi_getinfo(FI_VERSION(1, 5), NULL, NULL, 0, hints, &info);
     fi_freeinfo(hints);
 
+    fabric->info = NULL;
+
+    originfo = info;
+    useinfo = NULL;
+    while (info)
+    {
+        char *prov_name = info->fabric_attr->prov_name;
+        char *domain_name = info->domain_attr->name;
+
+        if (ifname && strcmp(ifname, domain_name) == 0)
+        {
+            useinfo = info;
+            break;
+        }
+        if (strcmp(prov_name, "verbs") == 0 || strcmp(prov_name, "gni") == 0 ||
+            strcmp(prov_name, "psm2") == 0)
+        {
+            useinfo = info;
+        }
+        info = info->next;
+    }
+
+    info = useinfo;
+
     if (!info)
     {
-        fabric->info = NULL;
         return;
     }
+
+    fabric->info = fi_dupinfo(info);
+
     if (info->mode & FI_CONTEXT2)
     {
         fabric->ctx = calloc(2, sizeof(*fabric->ctx));
@@ -142,16 +168,6 @@ static void init_fabric(struct fabric_state *fabric)
         fabric->msg_prefix_size = 0;
     }
 
-    while (info->next)
-    {
-        if (ifname && strcmp(ifname, info->domain_attr->name) == 0)
-        {
-            break;
-        }
-        info = info->next;
-    }
-
-    fabric->info = fi_dupinfo(info);
     fabric->addr_len = info->src_addrlen;
 
     info->domain_attr->mr_mode = FI_MR_BASIC;
@@ -175,7 +191,7 @@ static void init_fabric(struct fabric_state *fabric)
 
     fi_enable(fabric->signal);
 
-    fi_freeinfo(info);
+    fi_freeinfo(originfo);
 }
 
 static void fini_fabric(struct fabric_state *fabric)
@@ -1037,12 +1053,77 @@ static struct _CP_DP_Interface RdmaDPInterface;
  * greater than 1 if it can.  Eventually if we have more than one
  * possible RDMA DP, we may need some better scheme, maybe where the
  * "priority" return value represents some desirability measure
- * (like expected bandwidth or something).
+ * (like expected bandwidth or something)
+ *
+ * Returns 10 if a valid RDMA fabric is found, 100 if that fabric matches
+ * what is set in the FABRIC_IFACE environment variable.
+ *
  */
 static int RdmaGetPriority(CP_Services Svcs, void *CP_Stream,
                            struct _SstParams *Params)
 {
+    struct fi_info *hints, *info, *originfo, *useinfo;
+    char *ifname;
     int Ret = -1;
+
+    hints = fi_allocinfo();
+    hints->caps = FI_MSG | FI_SEND | FI_RECV | FI_REMOTE_READ |
+                  FI_REMOTE_WRITE | FI_RMA | FI_READ | FI_WRITE;
+    hints->mode = FI_CONTEXT | FI_LOCAL_MR | FI_CONTEXT2 | FI_MSG_PREFIX |
+                  FI_ASYNC_IOV | FI_RX_CQ_DATA;
+    hints->domain_attr->mr_mode = FI_MR_BASIC;
+
+    ifname = getenv("FABRIC_IFACE");
+
+    fi_getinfo(FI_VERSION(1, 5), NULL, NULL, 0, hints, &info);
+    fi_freeinfo(hints);
+
+    if (!info)
+    {
+        Svcs->verbose(CP_Stream,
+                      "RDMA Dataplane could not find any viable fabrics.\n");
+    }
+
+    originfo = info;
+
+    while (info)
+    {
+        char *prov_name, *domain_name;
+
+        prov_name = info->fabric_attr->prov_name;
+        domain_name = info->domain_attr->name;
+        if (ifname && strcmp(ifname, domain_name) == 0)
+        {
+            Svcs->verbose(CP_Stream, "RDMA Dataplane found the requested "
+                                     "interface %s, provider type %s.\n",
+                          ifname, prov_name);
+            Ret = 100;
+            break;
+        }
+        if (strcmp(prov_name, "verbs") == 0 || strcmp(prov_name, "gni") == 0 ||
+            strcmp(prov_name, "psm2") == 0)
+        {
+
+            Svcs->verbose(CP_Stream, "RDMA Dataplane sees interface %s, "
+                                     "provider type %s, which should work.\n",
+                          domain_name, prov_name);
+            Ret = 10;
+        }
+        info = info->next;
+    }
+
+    if (Ret == -1)
+    {
+        Svcs->verbose(
+            CP_Stream,
+            "RDMA Dataplane could not find an RDMA-compatible fabric.\n");
+    }
+
+    if (originfo)
+    {
+        fi_freeinfo(originfo);
+    }
+
     Svcs->verbose(
         CP_Stream,
         "RDMA Dataplane evaluating viability, returning priority %d\n", Ret);