prov/tcp: Add support for active TCP port range

There is a need (e.g. firewall rule) to limit TCP ports used for
active connections.

The use case is DOAS clients started by MPI accessing DAOS servers
behind firewall.  Since MPI and DAOS may not want to share the same
port restriction, add active port range support under domain object
using fi_set_val.

Define FI_TCP_DOMAIN_ACTIVE_PORT_RANGE to take a string argument
of "<low port number>-<high port number>".

Here is a sample call to use port range 5000 to 6000, inclusively:
fi_set_val(&domain->fid, FI_TCP_DOMAIN_ACTIVE_PORT_RANGE, "5000-6000");

To disable the range, set it to "0-0"
fi_set_val(&domain->fid, FI_TCP_DOMAIN_ACTIVE_PORT_RANGE, "0-0");

If the source address for an active connection contains a port
number, it will be used over a defined port range.

Signed-off-by: Chien Tin Tung <chien.tin.tung@intel.com>

include/rdma/fi_ext.h

@@ -78,6 +78,11 @@ enum {
 	FI_OPT_EFA_WRITE_IN_ORDER_ALIGNED_128_BYTES, /* bool */
 };
 
+enum {
+	/* null terminated string, <low port>-<high port> */
+	FI_TCP_DOMAIN_ACTIVE_PORT_RANGE = -FI_PROV_SPECIFIC_TCP,
+};
+
 struct fi_fid_export {
 	struct fid **fid;
 	uint64_t flags;

man/fi_tcp.7.md

@@ -90,6 +90,16 @@ show all environment variables defined for the tcp provider.
   through the standard socket APIs (i.e. connect, accept, send, recv).
   Default: disabled.
 
+The tcp provider also supports domain parameter via fi_set_val.
+
+*FI_TCP_DOMAIN_ACTIVE_PORT_RANGE*
+: Limit active connections to a TCP port range for a domain.  Value is
+  a string in "<low port number>-<high port number>" format.  Use
+  "0-0" to disable.  If an active connection's source address has a
+  port number, it will be used instead of the port range.
+  Sample:
+    fi_set_val(&domain->fid, FI_TCP_DOMAIN_ACTIVE_PORT_RANGE, "5000-6000");
+
 # NOTES
 
 The tcp provider supports both msg and rdm endpoints directly.  Support

prov/tcp/src/xnet.h

@@ -171,6 +171,8 @@ void xnet_connect_done(struct xnet_ep *ep);
 void xnet_req_done(struct xnet_ep *ep);
 int xnet_send_cm_msg(struct xnet_ep *ep);
 void xnet_uring_req_done(struct xnet_ep *ep, int res);
+int xnet_bind_to_port_range(SOCKET sock, void* src_addr, size_t addrlen,
+			    struct xnet_port_range *range);
 
 /* Inject buffer space is included */
 union xnet_hdrs {
@@ -471,6 +473,7 @@ struct xnet_domain {
 	struct util_domain		util_domain;
 	struct xnet_progress		progress;
 	enum fi_ep_type			ep_type;
+	struct xnet_port_range		active_ports;
 };
 
 static inline struct xnet_progress *xnet_ep2_progress(struct xnet_ep *ep)

prov/tcp/src/xnet_domain.c

@@ -199,11 +199,57 @@ static int xnet_domain_close(fid_t fid)
 	return FI_SUCCESS;
 }
 
+static int xnet_domain_set_val(struct xnet_domain *domain,
+			       int name,
+			       void *val)
+{
+	switch (name) {
+	case FI_TCP_DOMAIN_ACTIVE_PORT_RANGE:
+		if (strlen(val) > 11) /* 12345-78901 */
+			return -FI_EINVAL;
+		if (sscanf(val, "%d-%d", &domain->active_ports.low,
+			&domain->active_ports.high) != 2)
+			return -FI_EINVAL;
+
+		if (domain->active_ports.high > XNET_PORT_MAX_RANGE ||
+			domain->active_ports.low > domain->active_ports.high ||
+			domain->active_ports.high < 0 ||
+			domain->active_ports.low < 0)
+			return -FI_EINVAL;
+
+		break;
+	default:
+		return -FI_EINVAL;
+	}
+
+	return FI_SUCCESS;
+}
+
+static int xnet_domain_control(fid_t fid, int command, void *arg)
+{
+	struct xnet_domain *domain;
+
+	domain = container_of(fid, struct xnet_domain,
+			      util_domain.domain_fid.fid);
+
+	switch (command) {
+	case FI_SET_VAL:
+		return xnet_domain_set_val(domain,
+				((struct fi_fid_var *)arg)->name,
+				((struct fi_fid_var *)arg)->val);
+		break;
+	default:
+		return -FI_ENOSYS;
+	}
+
+	return FI_SUCCESS;
+}
+
 static struct fi_ops xnet_domain_fi_ops = {
 	.size = sizeof(struct fi_ops),
 	.close = xnet_domain_close,
 	.bind = ofi_domain_bind,
-	.control = fi_no_control,
+	.control = xnet_domain_control,
 	.ops_open = fi_no_ops_open,
 	.tostr = fi_no_tostr,
 	.ops_set = fi_no_ops_set,

prov/tcp/src/xnet_ep.c

@@ -718,6 +718,35 @@ static struct fi_ops_ep xnet_ep_ops = {
 	.tx_size_left = fi_no_tx_size_left,
 };
 
+static int xnet_bind_port(struct fid_domain *fid_domain, struct fi_info *info,
+			  struct xnet_ep *ep)
+{
+	struct xnet_domain *domain;
+
+	domain = container_of(fid_domain, struct xnet_domain,
+				util_domain.domain_fid);
+
+	if (info->src_addr && (!ofi_is_any_addr(info->src_addr) ||
+				ofi_addr_get_port(info->src_addr) ||
+				domain->active_ports.high)) {
+		/* port specified in src_addr takes precedence */
+		if (ofi_addr_get_port(info->src_addr))
+			return bind(ep->bsock.sock, info->src_addr,
+					(socklen_t) info->src_addrlen);
+		/* Check port range next */
+		if (domain->active_ports.high)
+			return xnet_bind_to_port_range(ep->bsock.sock,
+					info->src_addr, info->src_addrlen,
+					&domain->active_ports);
+		/* No port info */
+		xnet_set_no_port(ep->bsock.sock);
+		return bind(ep->bsock.sock, info->src_addr,
+				(socklen_t) info->src_addrlen);
+	}
+
+	return FI_SUCCESS;
+}
+
 int xnet_endpoint(struct fid_domain *domain, struct fi_info *info,
 		  struct fid_ep **ep_fid, void *context)
 {
@@ -783,20 +812,10 @@ int xnet_endpoint(struct fid_domain *domain, struct fi_info *info,
 		if (!xnet_io_uring)
 			xnet_set_zerocopy(ep->bsock.sock);
 
-		if (info->src_addr && (!ofi_is_any_addr(info->src_addr) ||
-					ofi_addr_get_port(info->src_addr))) {
-
-			if (!ofi_addr_get_port(info->src_addr)) {
-				xnet_set_no_port(ep->bsock.sock);
-			}
-
-			ret = bind(ep->bsock.sock, info->src_addr,
-				(socklen_t) info->src_addrlen);
-			if (ret) {
-				FI_WARN(&xnet_prov, FI_LOG_EP_CTRL, "bind failed\n");
-				ret = -ofi_sockerr();
-				goto err3;
-			}
+		if (xnet_bind_port(domain, info, ep)) {
+			FI_WARN(&xnet_prov, FI_LOG_EP_CTRL, "bind failed\n");
+			ret = -ofi_sockerr();
+			goto err3;
 		}
 	}
 

prov/tcp/src/xnet_pep.c

@@ -101,20 +101,23 @@ static struct fi_ops xnet_pep_fi_ops = {
 	.ops_open = fi_no_ops_open,
 };
 
-static int xnet_bind_to_port_range(SOCKET sock, void* src_addr, size_t addrlen)
+int xnet_bind_to_port_range(SOCKET sock, void* src_addr, size_t addrlen,
+			    struct xnet_port_range *range)
 {
-	int ret, i, rand_port_number;
+	int ret, i, rand_port_number, high, low;
 	static uint32_t seed;
+
+	high = range->high;
+	low = range->low;
 	if (!seed)
 		seed = ofi_generate_seed();
 
 	rand_port_number = ofi_xorshift_random_r(&seed) %
-			   (xnet_ports.high + 1 - xnet_ports.low) +
-			   xnet_ports.low;
+			   (high + 1 - low) + low;
 
-	for (i = xnet_ports.low; i <= xnet_ports.high; i++, rand_port_number++) {
-		if (rand_port_number > xnet_ports.high)
-			rand_port_number = xnet_ports.low;
+	for (i = low; i <= high; i++, rand_port_number++) {
+		if (rand_port_number > high)
+			rand_port_number = low;
 
 		ofi_addr_set_port(src_addr, (uint16_t) rand_port_number);
 		ret = bind(sock, src_addr, (socklen_t) addrlen);
@@ -129,7 +132,7 @@ static int xnet_bind_to_port_range(SOCKET sock, void* src_addr, size_t addrlen)
 		}
 		break;
 	}
-	return (i <= xnet_ports.high) ? FI_SUCCESS : -FI_EADDRNOTAVAIL;
+	return (i <= high) ? FI_SUCCESS : -FI_EADDRNOTAVAIL;
 }
 
 static int xnet_pep_sock_create(struct xnet_pep *pep)
@@ -175,7 +178,7 @@ static int xnet_pep_sock_create(struct xnet_pep *pep)
 			ret = -ofi_sockerr();
 	} else {
 		ret = xnet_bind_to_port_range(pep->sock, pep->info->src_addr,
-					      pep->info->src_addrlen);
+					      pep->info->src_addrlen, &xnet_ports);
 	}
 
 	if (ret) {