Merge pull request #1153 from yishaih/mlx5_misc

mlx5: Few enhancements including rping related fix

librdmacm/examples/rping.c

@@ -567,17 +567,33 @@ static int rping_create_qp(struct rping_cb *cb)
 	init_attr.qp_type = IBV_QPT_RC;
 	init_attr.send_cq = cb->cq;
 	init_attr.recv_cq = cb->cq;
+	id = cb->server ? cb->child_cm_id : cb->cm_id;
 
 	if (cb->self_create_qp) {
 		cb->qp = ibv_create_qp(cb->pd, &init_attr);
 		if (!cb->qp) {
 			perror("ibv_create_qp");
 			return -1;
 		}
-		return 0;
+
+		struct ibv_qp_attr attr = {
+			.qp_state = IBV_QPS_INIT,
+			.pkey_index = 0,
+			.port_num = id->port_num,
+			.qp_access_flags = 0,
+		};
+
+		ret = ibv_modify_qp(cb->qp, &attr,
+				    IBV_QP_STATE | IBV_QP_PKEY_INDEX |
+				    IBV_QP_PORT | IBV_QP_ACCESS_FLAGS);
+
+		if (ret) {
+			perror("ibv_modify_qp");
+			ibv_destroy_qp(cb->qp);
+		}
+		return ret ? -1 : 0;
 	}
 
-	id = cb->server ? cb->child_cm_id : cb->cm_id;
 	ret = rdma_create_qp(id, cb->pd, &init_attr);
 	if (!ret)
 		cb->qp = id->qp;

providers/mlx5/dbrec.c

@@ -39,7 +39,8 @@
 #include "mlx5.h"
 
 struct mlx5_db_page {
-	struct mlx5_db_page	       *prev, *next;
+	cl_map_item_t			cl_map;
+	struct list_node		available;
 	struct mlx5_buf			buf;
 	int				num_db;
 	int				use_cnt;
@@ -76,11 +77,9 @@ static struct mlx5_db_page *__add_page(struct mlx5_context *context)
 	for (i = 0; i < nlong; ++i)
 		page->free[i] = ~0;
 
-	page->prev = NULL;
-	page->next = context->db_list;
-	context->db_list = page;
-	if (page->next)
-		page->next->prev = page;
+	cl_qmap_insert(&context->dbr_map, (uintptr_t) page->buf.buf,
+		       &page->cl_map);
+	list_add(&context->dbr_available_pages, &page->available);
 
 	return page;
 }
@@ -110,18 +109,21 @@ __be32 *mlx5_alloc_dbrec(struct mlx5_context *context, struct ibv_pd *pd,
 	}
 
 default_alloc:
-	pthread_mutex_lock(&context->db_list_mutex);
+	pthread_mutex_lock(&context->dbr_map_mutex);
 
-	for (page = context->db_list; page; page = page->next)
-		if (page->use_cnt < page->num_db)
-			goto found;
+	page = list_top(&context->dbr_available_pages, struct mlx5_db_page,
+			available);
+	if (page)
+		goto found;
 
 	page = __add_page(context);
 	if (!page)
 		goto out;
 
 found:
 	++page->use_cnt;
+	if (page->use_cnt == page->num_db)
+		list_del(&page->available);
 
 	for (i = 0; !page->free[i]; ++i)
 		/* nothing */;
@@ -132,7 +134,7 @@ __be32 *mlx5_alloc_dbrec(struct mlx5_context *context, struct ibv_pd *pd,
 	db = page->buf.buf + (i * 8 * sizeof(long) + j) * context->cache_line_size;
 
 out:
-	pthread_mutex_unlock(&context->db_list_mutex);
+	pthread_mutex_unlock(&context->dbr_map_mutex);
 
 	return db;
 }
@@ -142,6 +144,7 @@ void mlx5_free_db(struct mlx5_context *context, __be32 *db, struct ibv_pd *pd,
 {
 	struct mlx5_db_page *page;
 	uintptr_t ps = to_mdev(context->ibv_ctx.context.device)->page_size;
+	cl_map_item_t *item;
 	int i;
 
 	if (custom_alloc) {
@@ -154,25 +157,21 @@ void mlx5_free_db(struct mlx5_context *context, __be32 *db, struct ibv_pd *pd,
 		return;
 	}
 
-	pthread_mutex_lock(&context->db_list_mutex);
+	pthread_mutex_lock(&context->dbr_map_mutex);
 
-	for (page = context->db_list; page; page = page->next)
-		if (((uintptr_t) db & ~(ps - 1)) == (uintptr_t) page->buf.buf)
-			break;
+	item = cl_qmap_get(&context->dbr_map, (uintptr_t) db & ~(ps - 1));
 
-	if (!page)
-		goto out;
+	assert(item != cl_qmap_end(&context->dbr_map));
 
+	page = (container_of(item, struct mlx5_db_page, cl_map));
 	i = ((void *) db - page->buf.buf) / context->cache_line_size;
 	page->free[i / (8 * sizeof(long))] |= 1UL << (i % (8 * sizeof(long)));
+	if (page->use_cnt == page->num_db)
+		list_add(&context->dbr_available_pages, &page->available);
 
 	if (!--page->use_cnt) {
-		if (page->prev)
-			page->prev->next = page->next;
-		else
-			context->db_list = page->next;
-		if (page->next)
-			page->next->prev = page->prev;
+		cl_qmap_remove_item(&context->dbr_map, item);
+		list_del(&page->available);
 
 		if (page->buf.type == MLX5_ALLOC_TYPE_EXTERNAL)
 			mlx5_free_buf_extern(context, &page->buf);
@@ -182,6 +181,5 @@ void mlx5_free_db(struct mlx5_context *context, __be32 *db, struct ibv_pd *pd,
 		free(page);
 	}
 
-out:
-	pthread_mutex_unlock(&context->db_list_mutex);
+	pthread_mutex_unlock(&context->dbr_map_mutex);
 }

providers/mlx5/mlx5.c

@@ -2394,9 +2394,10 @@ static int mlx5_set_context(struct mlx5_context *context,
 	for (i = 0; i < MLX5_MKEY_TABLE_SIZE; ++i)
 		context->mkey_table[i].refcnt = 0;
 
-	context->db_list = NULL;
+	list_head_init(&context->dbr_available_pages);
+	cl_qmap_init(&context->dbr_map);
 
-	pthread_mutex_init(&context->db_list_mutex, NULL);
+	pthread_mutex_init(&context->dbr_map_mutex, NULL);
 
 	context->prefer_bf = get_always_bf();
 	context->shut_up_bf = get_shut_up_bf();

providers/mlx5/mlx5.h

@@ -42,6 +42,7 @@
 
 #include <infiniband/driver.h>
 #include <util/udma_barrier.h>
+#include <util/cl_qmap.h>
 #include <util/util.h>
 #include "mlx5-abi.h"
 #include <util/bitmap.h>
@@ -345,8 +346,9 @@ struct mlx5_context {
 	pthread_mutex_t			mkey_table_mutex;
 
 	struct mlx5_uar_info		uar[MLX5_MAX_UARS];
-	struct mlx5_db_page	       *db_list;
-	pthread_mutex_t			db_list_mutex;
+	struct list_head		dbr_available_pages;
+	cl_qmap_t		        dbr_map;
+	pthread_mutex_t			dbr_map_mutex;
 	int				cache_line_size;
 	int				max_sq_desc_sz;
 	int				max_rq_desc_sz;
@@ -582,6 +584,11 @@ struct mlx5_wq {
 	unsigned			tail;
 	unsigned			cur_post;
 	int				max_gs;
+	/*
+	 * Equal to max_gs when qp is in RTS state for sq, or in INIT state for
+	 * rq, equal to -1 otherwise, used to verify qp_state in data path.
+	 */
+	int				qp_state_max_gs;
 	int				wqe_shift;
 	int				offset;
 	void			       *qend;
@@ -1273,6 +1280,8 @@ void mlx5_unimport_mr(struct ibv_mr *mr);
 struct ibv_pd *mlx5_import_pd(struct ibv_context *context,
 			      uint32_t pd_handle);
 void mlx5_unimport_pd(struct ibv_pd *pd);
+void mlx5_qp_fill_wr_complete_error(struct mlx5_qp *mqp);
+void mlx5_qp_fill_wr_complete_real(struct mlx5_qp *mqp);
 int mlx5_qp_fill_wr_pfns(struct mlx5_qp *mqp,
 			 const struct ibv_qp_init_attr_ex *attr,
 			 const struct mlx5dv_qp_init_attr *mlx5_attr);

providers/mlx5/qp.c

@@ -846,7 +846,7 @@ static inline int _mlx5_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr,
 			goto out;
 		}
 
-		if (unlikely(wr->num_sge > qp->sq.max_gs)) {
+		if (unlikely(wr->num_sge > qp->sq.qp_state_max_gs)) {
 			mlx5_dbg(fp, MLX5_DBG_QP_SEND, "max gs exceeded %d (max = %d)\n",
 				 wr->num_sge, qp->sq.max_gs);
 			err = ENOMEM;
@@ -1186,6 +1186,17 @@ static void mlx5_send_wr_start(struct ibv_qp_ex *ibqp)
 	mqp->inl_wqe = 0;
 }
 
+static int mlx5_send_wr_complete_error(struct ibv_qp_ex *ibqp)
+{
+	struct mlx5_qp *mqp = to_mqp((struct ibv_qp *)ibqp);
+
+	/* Rolling back */
+	mqp->sq.cur_post = mqp->cur_post_rb;
+	mqp->fm_cache = mqp->fm_cache_rb;
+	mlx5_spin_unlock(&mqp->sq.lock);
+	return EINVAL;
+}
+
 static int mlx5_send_wr_complete(struct ibv_qp_ex *ibqp)
 {
 	struct mlx5_qp *mqp = to_mqp((struct ibv_qp *)ibqp);
@@ -3453,6 +3464,22 @@ static void fill_wr_setters_eth(struct ibv_qp_ex *ibqp)
 	ibqp->wr_set_inline_data_list = mlx5_send_wr_set_inline_data_list_eth;
 }
 
+void mlx5_qp_fill_wr_complete_error(struct mlx5_qp *mqp)
+{
+	struct ibv_qp_ex *ibqp = &mqp->verbs_qp.qp_ex;
+
+	if (ibqp->wr_complete)
+		ibqp->wr_complete = mlx5_send_wr_complete_error;
+}
+
+void mlx5_qp_fill_wr_complete_real(struct mlx5_qp *mqp)
+{
+	struct ibv_qp_ex *ibqp = &mqp->verbs_qp.qp_ex;
+
+	if (ibqp->wr_complete)
+		ibqp->wr_complete = mlx5_send_wr_complete;
+}
+
 int mlx5_qp_fill_wr_pfns(struct mlx5_qp *mqp,
 			 const struct ibv_qp_init_attr_ex *attr,
 			 const struct mlx5dv_qp_init_attr *mlx5_attr)
@@ -3743,7 +3770,7 @@ int mlx5_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
 			goto out;
 		}
 
-		if (unlikely(wr->num_sge > qp->rq.max_gs)) {
+		if (unlikely(wr->num_sge > qp->rq.qp_state_max_gs)) {
 			err = EINVAL;
 			*bad_wr = wr;
 			goto out;

providers/mlx5/verbs.c

@@ -1893,7 +1893,7 @@ static int mlx5_alloc_qp_buf(struct ibv_context *context,
 	}
 
 	/* compatibility support */
-	qp_huge_key  = qptype2key(qp->ibv_qp->qp_type);
+	qp_huge_key = qptype2key(attr->qp_type);
 	if (mlx5_use_huge(qp_huge_key))
 		default_alloc_type = MLX5_ALLOC_TYPE_HUGE;
 
@@ -2247,6 +2247,27 @@ static int qp_init_wr_memcpy(struct mlx5_qp *mqp,
 	return reg_opaque_mr(attr->pd);
 }
 
+static void set_qp_operational_state(struct mlx5_qp *qp,
+				     enum ibv_qp_state state)
+{
+	switch (state) {
+	case IBV_QPS_RESET:
+		mlx5_qp_fill_wr_complete_error(qp);
+		qp->rq.qp_state_max_gs = -1;
+		qp->sq.qp_state_max_gs = -1;
+		break;
+	case IBV_QPS_INIT:
+		qp->rq.qp_state_max_gs = qp->rq.max_gs;
+		break;
+	case IBV_QPS_RTS:
+		qp->sq.qp_state_max_gs = qp->sq.max_gs;
+		mlx5_qp_fill_wr_complete_real(qp);
+		break;
+	default:
+		break;
+	}
+}
+
 static struct ibv_qp *create_qp(struct ibv_context *context,
 				struct ibv_qp_init_attr_ex *attr,
 				struct mlx5dv_qp_init_attr *mlx5_qp_attr)
@@ -2628,6 +2649,8 @@ static struct ibv_qp *create_qp(struct ibv_context *context,
 	if (attr->comp_mask & IBV_QP_INIT_ATTR_SEND_OPS_FLAGS)
 		qp->verbs_qp.comp_mask |= VERBS_QP_EX;
 
+	set_qp_operational_state(qp, IBV_QPS_RESET);
+
 	return ibqp;
 
 err_destroy:
@@ -3035,6 +3058,9 @@ int mlx5_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
 	    (attr->qp_state == IBV_QPS_INIT) && mqp->need_mmo_enable)
 		ret = qp_enable_mmo(qp);
 
+	if (!ret && (attr_mask & IBV_QP_STATE))
+		set_qp_operational_state(mqp, attr->qp_state);
+
 	return ret;
 }