prov/tcp: Handle case where epoll isn't natively supported

In the tcp provider, if wait support is requested, the code
calls ofi_dynpoll_get_fd() to retrieve the epoll fd.  That
fd can be passed into the progress epoll fd.  This works as
long as epoll support is available on the platform.

However, if epoll isn't available, ofi_dynpoll_get_fd() will
return an invalid fd.  We need to handle the non-epoll case
differently, as we can't rely on chaining the file descriptors
for event reporting.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>

prov/tcp/src/xnet_cq.c

@@ -279,7 +279,7 @@ int xnet_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
 	if (ret)
 		goto free_cq;
 
-	if (cq->util_cq.wait) {
+	if (cq->util_cq.wait && ofi_have_epoll) {
 		ret = ofi_wait_add_fd(cq->util_cq.wait,
 			       ofi_dynpoll_get_fd(&xnet_cq2_progress(cq)->epoll_fd),
 			       POLLIN, xnet_cq_wait_try_func, cq,
@@ -427,18 +427,10 @@ static int xnet_cntr_wait_try_func(void *arg)
 	return FI_SUCCESS;
 }
 
-static int xnet_cntr_add_progress(struct util_cntr *cntr,
-				  struct xnet_progress *progress,
-				  void *context)
-{
-	return ofi_wait_add_fd(cntr->wait,
-			       ofi_dynpoll_get_fd(&progress->epoll_fd),
-			       POLLIN, xnet_cntr_wait_try_func, NULL, context);
-}
-
 int xnet_cntr_open(struct fid_domain *fid_domain, struct fi_cntr_attr *attr,
 		   struct fid_cntr **cntr_fid, void *context)
 {
+	struct xnet_progress *progress;
 	struct xnet_domain *domain;
 	struct util_cntr *cntr;
 	struct fi_cntr_attr cntr_attr;
@@ -470,12 +462,15 @@ int xnet_cntr_open(struct fid_domain *fid_domain, struct fi_cntr_attr *attr,
 	if (attr->wait_obj == FI_WAIT_NONE) {
 		cntr->cntr_fid.ops = &xnet_cntr_ops;
 	} else {
-		if (attr->wait_obj == FI_WAIT_FD && ofi_have_epoll)
-			ret = xnet_cntr_add_progress(cntr,
-						     xnet_cntr2_progress(cntr),
-						     &cntr->cntr_fid);
-		else
-			ret = xnet_start_progress(xnet_cntr2_progress(cntr));
+		progress = xnet_cntr2_progress(cntr);
+		if (attr->wait_obj == FI_WAIT_FD && ofi_have_epoll) {
+			ret = ofi_wait_add_fd(cntr->wait,
+					ofi_dynpoll_get_fd(&progress->epoll_fd),
+					POLLIN, xnet_cntr_wait_try_func, NULL,
+					&cntr->cntr_fid);
+		} else {
+			ret = xnet_start_progress(progress);
+		}
 		if (ret)
 			goto cleanup;
 	}

prov/tcp/src/xnet_eq.c

@@ -141,7 +141,7 @@ static void xnet_eq_del_domain(struct xnet_eq *eq, struct xnet_domain *domain)
 	fid_list_remove(&eq->domain_list, NULL,
 			&domain->util_domain.domain_fid.fid);
 
-	if (eq->util_eq.wait) {
+	if (eq->util_eq.wait && ofi_have_epoll) {
 		(void) ofi_wait_del_fd(eq->util_eq.wait,
 				ofi_dynpoll_get_fd(&domain->progress.epoll_fd));
 	}
@@ -175,7 +175,7 @@ int xnet_add_domain_progress(struct xnet_eq *eq, struct xnet_domain *domain)
 	if (ret)
 		goto unlock;
 
-	if (eq->util_eq.wait) {
+	if (eq->util_eq.wait && ofi_have_epoll) {
 		ret = ofi_wait_add_fd(eq->util_eq.wait,
 				ofi_dynpoll_get_fd(&domain->progress.epoll_fd),
 				POLLIN, xnet_eq_wait_try_func, NULL, domain);
@@ -221,11 +221,13 @@ int xnet_eq_create(struct fid_fabric *fabric_fid, struct fi_eq_attr *attr,
 		goto err3;
 
 	if (eq->util_eq.wait) {
-		ret = ofi_wait_add_fd(eq->util_eq.wait,
-				ofi_dynpoll_get_fd(&eq->progress.epoll_fd),
-				POLLIN, xnet_eq_wait_try_func, NULL, eq);
-		if (ret)
-			goto err4;
+		if (ofi_have_epoll) {
+			ret = ofi_wait_add_fd(eq->util_eq.wait,
+					ofi_dynpoll_get_fd(&eq->progress.epoll_fd),
+					POLLIN, xnet_eq_wait_try_func, NULL, eq);
+			if (ret)
+				goto err4;
+		}
 
 		if (eq->util_eq.wait->wait_obj != FI_WAIT_FD || !ofi_have_epoll) {
 			ret = xnet_start_progress(&eq->progress);
@@ -244,8 +246,10 @@ int xnet_eq_create(struct fid_fabric *fabric_fid, struct fi_eq_attr *attr,
 	return 0;
 
 err5:
-	ofi_wait_del_fd(eq->util_eq.wait,
-			ofi_dynpoll_get_fd(&eq->progress.epoll_fd));
+	if (ofi_have_epoll) {
+		ofi_wait_del_fd(eq->util_eq.wait,
+				ofi_dynpoll_get_fd(&eq->progress.epoll_fd));
+	}
 err4:
 	xnet_close_progress(&eq->progress);
 err3: