Merge pull request #5 from j-xiong/master

various small fixes for psm provider, plus add missing definition of FI_MIN_MULTI_RECV.

include/rdma/fi_endpoint.h

@@ -100,6 +100,7 @@ enum {
 	FI_OPT_TOTAL_BUFFERED_RECV,	/* size_t */
 	FI_OPT_MAX_MSG_SIZE,		/* size_t */
 	FI_OPT_DATA_FLOW,		/* int */
+	FI_OPT_MIN_MULTI_RECV,		/* size_t */
 };
 
 struct fi_ops_ep {

man/fi_endpoint.3

@@ -319,6 +319,9 @@ single operation.
 .IP "FI_OPT_DATA_FLOW - int"
 Sets the default data flow associated with an endpoint or alias.  Data
 flows are referenced by index, from 0 to the endpoint attribute data_flow_cnt.
+.IP "FI_OPT_MIN_MULTI_RECV - size_t"
+Defines the minimum receive buffer space available when the receive buffer
+is automatically freed (see FI_MULTI_RECV).
 .RE
 .SH "CONTROL FLAGS"
 The following flags may be set to control the operation of data transfers

man/fi_msg.3

@@ -198,8 +198,8 @@ consumed.  Use of this flag may cause a single posted receive operation
 to generate multiple events as messages are placed into the buffer.
 The placement of received data into the buffer may be subjected to
 provider specific alignment restrictions.  The buffer will be freed from
-the endpoint when a message is received that cannot fit into the remaining
-free buffer space.
+the endpoint when the available buffer space falls below the network's
+MTU size (see FI_OPT_MIN_MULTI_RECV).
 .IP "FI_REMOTE_COMPLETE"
 Applies to fi_sendmsg.  Indicates that a completion should not be generated
 until the operation has completed on the remote side.

prov/psm/src/psmx.h

@@ -99,6 +99,7 @@ struct psmx_fid_domain {
 	psm_mq_t		psm_mq;
 	pthread_t		ns_thread;
 	int			ns_port;
+	uint64_t		ep_cap;
 
 	/* certain bits in the tag space can be reserved for non tag-matching
 	 * purpose. The tag-matching functions automatically treat these bits
@@ -248,7 +249,6 @@ struct psmx_fid_ep {
 struct psmx_fid_mr {
 	struct fid_mr		mr;
 	struct psmx_fid_domain	*domain;
-	struct psmx_fid_ep	*ep;
 	struct psmx_fid_eq	*eq;
 	struct psmx_fid_cntr	*cntr;
 	uint64_t		access;

prov/psm/src/psmx_domain.c

@@ -56,28 +56,9 @@ static int psmx_domain_close(fid_t fid)
 	return 0;
 }
 
-/* TODO: check on this call, an active EP should be bound to a domain on creation? */
 static int psmx_domain_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
 {
-	int err;
-
-	if (!bfid)
-		return -EINVAL;
-
-	switch (bfid->fclass) {
-	case FID_CLASS_EP:
-		if (!bfid->ops || !bfid->ops->bind)
-			return -EINVAL;
-		err = bfid->ops->bind(bfid, fid, flags);
-		if (err)
-			return err;
-		break;
-
-	default:
-		return -ENOSYS;
-	}
-
-	return 0;
+	return -ENOSYS;
 }
 
 static int psmx_domain_sync(fid_t fid, uint64_t flags, void *context)
@@ -177,6 +158,8 @@ int psmx_domain_open(struct fid_fabric *fabric, struct fi_info *info,
 	if (info->ep_cap & FI_MSG)
 		fid_domain->reserved_tag_bits |= PSMX_MSG_BIT;
 
+	fid_domain->ep_cap = info->ep_cap;
+
 	*domain = &fid_domain->domain;
 
 	return 0;

prov/psm/src/psmx_ep.c

@@ -164,7 +164,6 @@ static int psmx_ep_close(fid_t fid)
 static int psmx_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
 {
 	struct psmx_fid_ep *fid_ep;
-	struct psmx_fid_domain *domain;
 	struct psmx_fid_av *av;
 	struct psmx_fid_eq *eq;
 	struct psmx_fid_cntr *cntr;
@@ -175,13 +174,6 @@ static int psmx_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
 	if (!bfid)
 		return -EINVAL;
 	switch (bfid->fclass) {
-	case FID_CLASS_DOMAIN:
-		domain = container_of(bfid, struct psmx_fid_domain, domain.fid);
-		if (fid_ep->domain && fid_ep->domain != domain)
-			return -EEXIST;
-		fid_ep->domain = domain;
-		break;
-
 	case FID_CLASS_EQ:
 		eq = container_of(bfid, struct psmx_fid_eq, eq.fid);
 		if (flags & (FI_SEND | FI_READ | FI_WRITE)) {
@@ -192,7 +184,7 @@ static int psmx_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
 			if (fid_ep->recv_eq && fid_ep->recv_eq != eq)
 				return -EEXIST;
 		}
-		if (fid_ep->domain && fid_ep->domain != eq->domain)
+		if (fid_ep->domain != eq->domain)
 			return -EINVAL;
 		if (flags & (FI_SEND | FI_READ | FI_WRITE)) {
 			fid_ep->send_eq = eq;
@@ -204,7 +196,6 @@ static int psmx_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
 			if (flags & FI_EVENT)
 				fid_ep->recv_eq_event_flag = 1;
 		}
-		fid_ep->domain = eq->domain;
 		break;
 
 	case FID_CLASS_CNTR:
@@ -225,7 +216,7 @@ static int psmx_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
 			if (fid_ep->read_cntr && fid_ep->read_cntr != cntr)
 				return -EEXIST;
 		}
-		if (fid_ep->domain && fid_ep->domain != cntr->domain)
+		if (fid_ep->domain != cntr->domain)
 			return -EINVAL;
 		if (flags & FI_SEND) {
 			fid_ep->send_cntr = cntr;
@@ -247,18 +238,16 @@ static int psmx_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
 			if (flags & FI_EVENT)
 				fid_ep->read_cntr_event_flag = 1;
 		}
-		fid_ep->domain = cntr->domain;
 		break;
 
 	case FID_CLASS_AV:
 		av = container_of(bfid,
 				struct psmx_fid_av, av.fid);
 		if (fid_ep->av && fid_ep->av != av)
 			return -EEXIST;
-		if (fid_ep->domain && fid_ep->domain != av->domain)
+		if (fid_ep->domain != av->domain)
 			return -EINVAL;
 		fid_ep->av = av;
-		fid_ep->domain = av->domain;
 		break;
 
 	case FID_CLASS_MR:
@@ -361,8 +350,16 @@ static struct fi_ops_ep psmx_ep_ops = {
 int psmx_ep_open(struct fid_domain *domain, struct fi_info *info,
 		struct fid_ep **ep, void *context)
 {
+	struct psmx_fid_domain *fid_domain;
 	struct psmx_fid_ep *fid_ep;
 
+	fid_domain = container_of(domain, struct psmx_fid_domain, domain.fid);
+	if (!fid_domain)
+		return -EINVAL;
+
+	if (info && ((info->ep_cap & fid_domain->ep_cap) != info->ep_cap))
+		return -EINVAL;
+
 	fid_ep = (struct psmx_fid_ep *) calloc(1, sizeof *fid_ep);
 	if (!fid_ep)
 		return -ENOMEM;
@@ -373,6 +370,8 @@ int psmx_ep_open(struct fid_domain *domain, struct fi_info *info,
 	fid_ep->ep.ops = &psmx_ep_ops;
 	fid_ep->ep.cm = &psmx_cm_ops;
 	fid_ep->ep.tagged = &psmx_tagged_ops;
+	fid_ep->domain = fid_domain;
+
 	PSMX_CTXT_TYPE(&fid_ep->nocomp_send_context) = PSMX_NOCOMP_SEND_CONTEXT;
 	PSMX_CTXT_EP(&fid_ep->nocomp_send_context) = fid_ep;
 	PSMX_CTXT_TYPE(&fid_ep->nocomp_recv_context) = PSMX_NOCOMP_RECV_CONTEXT;

prov/psm/src/psmx_mr.c

@@ -161,11 +161,8 @@ static int psmx_mr_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
 	switch (bfid->fclass) {
 	case FID_CLASS_EP:
 		ep = container_of(bfid, struct psmx_fid_ep, ep.fid);
-		if (fid_mr->ep && fid_mr->ep != ep)
-			return -EEXIST;
 		if (fid_mr->domain != ep->domain)
 			return -EINVAL;
-		fid_mr->ep = ep;
 		break;
 
 	case FID_CLASS_EQ:

prov/psm/src/psmx_tagged.c

@@ -44,6 +44,11 @@ ssize_t _psmx_tagged_recvfrom(struct fid_ep *ep, void *buf, size_t len,
 	int user_fi_context = 0;
 	int err;
 
+	if (src_addr) {
+		fprintf(stderr, "%s: error: non-NULL src_addr is not supported.", __func__);
+		return -EINVAL;
+	}
+
 	if (flags & FI_TRIGGER) {
 		struct psmx_trigger *trigger;
 		struct fi_triggered_context *ctxt = context;