efa: Respect minimum SQ size

The min SQ size reports the minimum number of WQEs per SQ. Respect the min SQ size on QP creation. Signed-off-by: Gal Pressman <galpress@amazon.com>
linux-rdma · Aug 2, 2020 · 0e6660c · 0e6660c
1 parent 7aad28d
commit 0e6660c
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 3 deletions.
diff --git a/providers/efa/efa.c b/providers/efa/efa.c
@@ -58,6 +58,7 @@ static struct verbs_context *efa_alloc_context(struct ibv_device *vdev,
 	int err;
 
 	cmd.comp_mask |= EFA_ALLOC_UCONTEXT_CMD_COMP_TX_BATCH;
+	cmd.comp_mask |= EFA_ALLOC_UCONTEXT_CMD_COMP_MIN_SQ_WR;
 
 	ctx = verbs_init_and_alloc_context(vdev, cmd_fd, ctx, ibvctx,
 					   RDMA_DRIVER_EFA);
@@ -74,6 +75,7 @@ static struct verbs_context *efa_alloc_context(struct ibv_device *vdev,
 	ctx->inline_buf_size = resp.inline_buf_size;
 	ctx->max_llq_size = resp.max_llq_size;
 	ctx->max_tx_batch = resp.max_tx_batch;
+	ctx->min_sq_wr = resp.min_sq_wr;
 	pthread_spin_init(&ctx->qp_table_lock, PTHREAD_PROCESS_PRIVATE);
 
 	/* ah udata is mandatory for ah number retrieval */

diff --git a/providers/efa/efa.h b/providers/efa/efa.h
@@ -30,6 +30,7 @@ struct efa_context {
 	uint32_t max_rdma_size;
 	uint16_t max_wr_rdma_sge;
 	uint16_t max_tx_batch;
+	uint16_t min_sq_wr;
 	size_t cqe_size;
 	struct efa_qp **qp_table;
 	unsigned int qp_table_sz_m1;

diff --git a/providers/efa/verbs.c b/providers/efa/verbs.c
@@ -621,6 +621,13 @@ static int efa_sq_initialize(struct efa_qp *qp,
 	sq->max_batch_wr = ctx->max_tx_batch ?
 		(ctx->max_tx_batch * 64) / sizeof(struct efa_io_tx_wqe) :
 		UINT16_MAX;
+	if (ctx->min_sq_wr) {
+		/* The device can't accept a doorbell for the whole SQ at once,
+		 * set the max batch to at least (SQ size - 1).
+		 */
+		sq->max_batch_wr = min_t(uint32_t, sq->max_batch_wr,
+					 sq->wq.wqe_cnt - 1);
+	}
 
 	return 0;
 
@@ -692,15 +699,17 @@ static void efa_qp_init_indices(struct efa_qp *qp)
 	qp->rq.wq.wrid_idx_pool_next = 0;
 }
 
-static void efa_setup_qp(struct efa_qp *qp,
+static void efa_setup_qp(struct efa_context *ctx,
+			 struct efa_qp *qp,
 			 struct ibv_qp_cap *cap,
 			 size_t page_size)
 {
 	uint16_t rq_desc_cnt;
 
 	efa_qp_init_indices(qp);
 
-	qp->sq.wq.wqe_cnt = roundup_pow_of_two(cap->max_send_wr);
+	qp->sq.wq.wqe_cnt = roundup_pow_of_two(max_t(uint32_t, cap->max_send_wr,
+						     ctx->min_sq_wr));
 	qp->sq.wq.max_sge = cap->max_send_sge;
 	qp->sq.wq.desc_mask = qp->sq.wq.wqe_cnt - 1;
 
@@ -840,7 +849,7 @@ static struct ibv_qp *create_qp(struct ibv_context *ibvctx,
 		goto err_out;
 	}
 
-	efa_setup_qp(qp, &attr->cap, dev->pg_sz);
+	efa_setup_qp(ctx, qp, &attr->cap, dev->pg_sz);
 
 	attr->cap.max_send_wr = qp->sq.wq.wqe_cnt;
 	attr->cap.max_recv_wr = qp->rq.wq.wqe_cnt;