prov/gni: fix a problem with progress

Turns out that the VC refactor had another impact,
it causes all data progress to be delayed until
the app starts calling fi_cq_read, etc.  This
brings out all kind of issues with one-sided
program models.

Signed-off-by: Howard Pritchard <howardp@lanl.gov>

prov/gni/src/gnix_atomic.c

@@ -665,9 +665,16 @@ ssize_t _gnix_atomic(struct gnix_fid_ep *ep,
 
 	COND_RELEASE(ep->requires_lock, &ep->vc_lock);
 
-	/* If a new VC was allocated, progress CM before returning. */
-	if (!connected)
+	/*
+	 *If a new VC was allocated, progress CM before returning.
+	 * If the VC is connected and there's a backlog, poke
+	 * the nic progress engine befure returning.
+	 */
+	if (!connected) {
 		_gnix_cm_nic_progress(ep->cm_nic);
+	} else if (!dlist_empty(&vc->tx_queue)) {
+		_gnix_nic_progress(vc->ep->nic);
+	}
 
 	return rc;
 

prov/gni/src/gnix_msg.c

@@ -3061,9 +3061,16 @@ ssize_t _gnix_send(struct gnix_fid_ep *ep, uint64_t loc_addr, size_t len,
 
 	COND_RELEASE(vc->ep->requires_lock, &vc->ep->vc_lock);
 
-	/* If a new VC was allocated, progress CM before returning. */
-	if (!connected)
+	/*
+	 * If a new VC was allocated, progress CM before returning.
+	 * If the VC is connected and there's a backlog, poke
+	 * the nic progress engine befure returning.
+	 */
+	if (!connected) {
 		_gnix_cm_nic_progress(ep->cm_nic);
+	} else if (!dlist_empty(&vc->tx_queue)) {
+		_gnix_nic_progress(vc->ep->nic);
+	}
 
 	return ret;
 
@@ -3515,9 +3522,16 @@ ssize_t _gnix_sendv(struct gnix_fid_ep *ep, const struct iovec *iov,
 
 	COND_RELEASE(ep->requires_lock, &ep->vc_lock);
 
-	/* If a new VC was allocated, progress CM before returning. */
-	if (!connected)
+	/*
+	 * If a new VC was allocated, progress CM before returning.
+	 * If the VC is connected and there's a backlog, poke
+	 * the nic progress engine befure returning.
+	 */
+	if (!connected) {
 		_gnix_cm_nic_progress(ep->cm_nic);
+	} else if (!dlist_empty(&vc->tx_queue)) {
+		_gnix_nic_progress(vc->ep->nic);
+	}
 
 	return ret;
 

prov/gni/src/gnix_rma.c

@@ -1516,9 +1516,16 @@ ssize_t _gnix_rma(struct gnix_fid_ep *ep, enum gnix_fab_req_type fr_type,
 
 	COND_RELEASE(ep->requires_lock, &ep->vc_lock);
 
-	/* If a new VC was allocated, progress CM before returning. */
-	if (!connected)
+	/*
+	 * If a new VC was allocated, progress CM before returning.
+	 * If the VC is connected and there's a backlog, poke
+	 * the nic progress engine befure returning.
+	 */
+	if (!connected) {
 		_gnix_cm_nic_progress(ep->cm_nic);
+	} else if (!dlist_empty(&vc->tx_queue)) {
+		_gnix_nic_progress(vc->ep->nic);
+	}
 
 	return rc;
 

prov/gni/src/gnix_vc.c

@@ -1925,9 +1925,6 @@ int _gnix_vc_queue_tx_req(struct gnix_fab_req *req)
 	}
 
 	if (unlikely(queue_tx)) {
-		/*
-		 * TODO: for auto progress do something here
-		 */
 		dlist_insert_tail(&req->dlist, &vc->tx_queue);
 		_gnix_vc_tx_schedule(vc);
 	}