From a95f62b7e42f543596db20f6fdf72468d9d16f1b Mon Sep 17 00:00:00 2001 From: Jiaran Zhang Date: Sun, 26 Apr 2020 10:26:06 +0800 Subject: [PATCH 1/3] libhns: Check number of extended sge when using extended atomic A WR should only be allowed to use the extended sge space allocated for itself, extended atomic operation is no exception. So add a check for this before filling extended atomic segment to ensure that the WR has enough number of extended sges. Signed-off-by: Jiaran Zhang Signed-off-by: Weihang Li --- providers/hns/hns_roce_u_hw_v2.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/providers/hns/hns_roce_u_hw_v2.c b/providers/hns/hns_roce_u_hw_v2.c index b95017def..198b9ffa3 100644 --- a/providers/hns/hns_roce_u_hw_v2.c +++ b/providers/hns/hns_roce_u_hw_v2.c @@ -84,6 +84,10 @@ static int set_atomic_seg(struct hns_roce_qp *qp, struct ibv_send_wr *wr, ext_sg_num = msg_len * DATA_TYPE_NUM >> HNS_ROCE_SGE_SHIFT; aseg->fetchadd_swap_data = 0; aseg->cmp_data = 0; + + if (ext_sg_num + HNS_ROCE_SGE_IN_WQE > qp->sq.max_gs) + return EINVAL; + if (wr->opcode == IBV_WR_ATOMIC_CMP_AND_SWP) { if (!wr->wr.atomic.swap || !wr->wr.atomic.compare_add) return EINVAL; From 6942d696fbb16a0647e46bc4e987708330cf592a Mon Sep 17 00:00:00 2001 From: Wenpeng Liang Date: Thu, 23 Apr 2020 15:07:23 +0800 Subject: [PATCH 2/3] libhns: Remove redundant parameters of two SRQ related functions After the content of the function is changed, some redundant parameters should be removed. Signed-off-by: Wenpeng Liang Signed-off-by: Weihang Li --- providers/hns/hns_roce_u_verbs.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/providers/hns/hns_roce_u_verbs.c b/providers/hns/hns_roce_u_verbs.c index 2020f555b..803fe61e7 100644 --- a/providers/hns/hns_roce_u_verbs.c +++ b/providers/hns/hns_roce_u_verbs.c @@ -390,7 +390,7 @@ int hns_roce_u_destroy_cq(struct ibv_cq *cq) return ret; } -static int hns_roce_create_idx_que(struct ibv_pd *pd, struct hns_roce_srq *srq) +static int hns_roce_create_idx_que(struct hns_roce_srq *srq) { struct hns_roce_idx_que *idx_que = &srq->idx_que; unsigned int buf_size; @@ -417,8 +417,7 @@ static int hns_roce_create_idx_que(struct ibv_pd *pd, struct hns_roce_srq *srq) return 0; } -static int hns_roce_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr, - struct hns_roce_srq *srq) +static int hns_roce_alloc_srq_buf(struct hns_roce_srq *srq) { int srq_buf_size; @@ -464,16 +463,13 @@ struct ibv_srq *hns_roce_u_create_srq(struct ibv_pd *pd, srq->wqe_cnt = roundup_pow_of_two(init_attr->attr.max_wr + 1); srq->max_gs = init_attr->attr.max_sge; - ret = hns_roce_create_idx_que(pd, srq); - if (ret) { - fprintf(stderr, "hns_roce_create_idx_que failed!\n"); + ret = hns_roce_create_idx_que(srq); + if (ret) goto out; - } - if (hns_roce_alloc_srq_buf(pd, &init_attr->attr, srq)) { - fprintf(stderr, "hns_roce_alloc_srq_buf failed!\n"); + ret = hns_roce_alloc_srq_buf(srq); + if (ret) goto err_idx_que; - } srq->db = hns_roce_alloc_db(to_hr_ctx(pd->context), HNS_ROCE_QP_TYPE_DB); From 1fe4b2ecc42771dcb8e6283ae950268d42fb8d85 Mon Sep 17 00:00:00 2001 From: Lang Cheng Date: Wed, 27 May 2020 14:06:17 +0800 Subject: [PATCH 3/3] libhns: Optimize the offset information of mmap areas of device In multiple memory mapped areas under the device context, the offset information is sequentially dependent on each other. It should be defined as a variable instead of using a fixed value. Signed-off-by: Lang Cheng Signed-off-by: Weihang Li --- providers/hns/hns_roce_u.c | 26 ++++++++++++-------------- providers/hns/hns_roce_u.h | 1 - 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/providers/hns/hns_roce_u.c b/providers/hns/hns_roce_u.c index e5b9488c4..e93f82560 100644 --- a/providers/hns/hns_roce_u.c +++ b/providers/hns/hns_roce_u.c @@ -90,12 +90,13 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev, int cmd_fd, void *private_data) { - int i; - struct ibv_get_context cmd; + struct hns_roce_device *hr_dev = to_hr_dev(ibdev); + struct hns_roce_alloc_ucontext_resp resp = {}; struct ibv_device_attr dev_attrs; struct hns_roce_context *context; - struct hns_roce_alloc_ucontext_resp resp = {}; - struct hns_roce_device *hr_dev = to_hr_dev(ibdev); + struct ibv_get_context cmd; + int offset = 0; + int i; context = verbs_init_and_alloc_context(ibdev, cmd_fd, context, ibv_ctx, RDMA_DRIVER_HNS); @@ -115,12 +116,12 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev, for (i = 0; i < HNS_ROCE_QP_TABLE_SIZE; ++i) context->qp_table[i].refcnt = 0; - context->uar = mmap(NULL, hr_dev->page_size, - PROT_READ | PROT_WRITE, MAP_SHARED, cmd_fd, 0); - if (context->uar == MAP_FAILED) { - fprintf(stderr, PFX "Warning: failed to mmap() uar page.\n"); + context->uar = mmap(NULL, hr_dev->page_size, PROT_READ | PROT_WRITE, + MAP_SHARED, cmd_fd, offset); + if (context->uar == MAP_FAILED) goto err_free; - } + + offset += hr_dev->page_size; if (hr_dev->hw_version == HNS_ROCE_HW_VER1) { /* @@ -129,12 +130,9 @@ static struct verbs_context *hns_roce_alloc_context(struct ibv_device *ibdev, */ context->cq_tptr_base = mmap(NULL, HNS_ROCE_CQ_DB_BUF_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, - cmd_fd, HNS_ROCE_TPTR_OFFSET); - if (context->cq_tptr_base == MAP_FAILED) { - fprintf(stderr, - PFX "Warning: Failed to mmap cq_tptr page.\n"); + cmd_fd, offset); + if (context->cq_tptr_base == MAP_FAILED) goto db_free; - } } pthread_spin_init(&context->uar_lock, PTHREAD_PROCESS_PRIVATE); diff --git a/providers/hns/hns_roce_u.h b/providers/hns/hns_roce_u.h index e39642fce..dc8935d42 100644 --- a/providers/hns/hns_roce_u.h +++ b/providers/hns/hns_roce_u.h @@ -71,7 +71,6 @@ #define HNS_ROCE_GID_SIZE 16 #define HNS_ROCE_CQ_DB_BUF_SIZE ((HNS_ROCE_MAX_CQ_NUM >> 11) << 12) -#define HNS_ROCE_TPTR_OFFSET 0x1000 #define HNS_ROCE_STATIC_RATE 3 /* Gbps */ #define HNS_ROCE_ADDRESS_MASK 0xFFFFFFFF