Skip to content

Commit

Permalink
Merge pull request #775 from amzn/ucontext-attr-pr
Browse files Browse the repository at this point in the history 
EFA ucontext attributes
  • Loading branch information
@rleon
rleon committed Jun 23, 2020
2 parents f18feb6 + eee1a96 commit d800c8a
Show file tree
Hide file tree
Showing 2 changed files with 33 additions and 35 deletions.
18 changes: 9 additions & 9 deletions providers/efa/efa.h
View file
Expand Up @@ -22,6 +22,13 @@ struct efa_context {
uint16_t sub_cqs_per_cq;
uint16_t inline_buf_size;
uint32_t max_llq_size;
uint32_t device_caps;
uint32_t max_sq_wr;
uint32_t max_rq_wr;
uint16_t max_sq_sge;
uint16_t max_rq_sge;
uint32_t max_rdma_size;
uint16_t max_wr_rdma_sge;
size_t cqe_size;
struct efa_qp **qp_table;
unsigned int qp_table_sz_m1;
Expand Down Expand Up @@ -125,18 +132,11 @@ struct efa_ah {
struct efa_dev {
struct verbs_device vdev;
uint32_t pg_sz;
uint32_t device_caps;
uint32_t max_sq_wr;
uint32_t max_rq_wr;
uint16_t max_sq_sge;
uint16_t max_rq_sge;
uint32_t max_rdma_size;
uint16_t max_wr_rdma_sge;
};

static inline bool is_rdma_read_cap(struct efa_dev *dev)
static inline bool is_rdma_read_cap(struct efa_context *ctx)
{
return dev->device_caps & EFA_QUERY_DEVICE_CAPS_RDMA_READ;
return ctx->device_caps & EFA_QUERY_DEVICE_CAPS_RDMA_READ;
}

static inline struct efa_dev *to_efa_dev(struct ibv_device *ibvdev)
Expand Down
50 changes: 24 additions & 26 deletions providers/efa/verbs.c
View file
Expand Up @@ -82,7 +82,6 @@ int efa_query_device_ex(struct ibv_context *context,
size_t attr_size)
{
struct efa_context *ctx = to_efa_context(context);
struct efa_dev *dev = to_efa_dev(context->device);
int cmd_supp_uhw = ctx->cmds_supp_udata_mask &
EFA_USER_CMDS_SUPP_UDATA_QUERY_DEVICE;
struct ibv_device_attr *a = &attr->orig_attr;
Expand All @@ -98,13 +97,13 @@ int efa_query_device_ex(struct ibv_context *context,
if (err)
return err;

dev->device_caps = resp.device_caps;
dev->max_sq_wr = resp.max_sq_wr;
dev->max_rq_wr = resp.max_rq_wr;
dev->max_sq_sge = resp.max_sq_sge;
dev->max_rq_sge = resp.max_rq_sge;
dev->max_rdma_size = resp.max_rdma_size;
dev->max_wr_rdma_sge = a->max_sge_rd;
ctx->device_caps = resp.device_caps;
ctx->max_sq_wr = resp.max_sq_wr;
ctx->max_rq_wr = resp.max_rq_wr;
ctx->max_sq_sge = resp.max_sq_sge;
ctx->max_rq_sge = resp.max_rq_sge;
ctx->max_rdma_size = resp.max_rdma_size;
ctx->max_wr_rdma_sge = a->max_sge_rd;

a->max_qp_wr = min_t(int, a->max_qp_wr,
ctx->max_llq_size / sizeof(struct efa_io_tx_wqe));
Expand All @@ -119,7 +118,6 @@ int efadv_query_device(struct ibv_context *ibvctx,
uint32_t inlen)
{
struct efa_context *ctx = to_efa_context(ibvctx);
struct efa_dev *dev = to_efa_dev(ibvctx->device);
uint64_t comp_mask_out = 0;

if (!is_efa_dev(ibvctx->device))
Expand All @@ -129,16 +127,16 @@ int efadv_query_device(struct ibv_context *ibvctx,
return EINVAL;

memset(attr, 0, inlen);
attr->max_sq_wr = dev->max_sq_wr;
attr->max_rq_wr = dev->max_rq_wr;
attr->max_sq_sge = dev->max_sq_sge;
attr->max_rq_sge = dev->max_rq_sge;
attr->max_sq_wr = ctx->max_sq_wr;
attr->max_rq_wr = ctx->max_rq_wr;
attr->max_sq_sge = ctx->max_sq_sge;
attr->max_rq_sge = ctx->max_rq_sge;
attr->inline_buf_size = ctx->inline_buf_size;

if (vext_field_avail(typeof(*attr), max_rdma_size, inlen)) {
attr->max_rdma_size = dev->max_rdma_size;
attr->max_rdma_size = ctx->max_rdma_size;

if (is_rdma_read_cap(dev))
if (is_rdma_read_cap(ctx))
attr->device_caps |= EFADV_DEVICE_ATTR_CAPS_RDMA_READ;
}

Expand Down Expand Up @@ -570,7 +568,7 @@ static int efa_sq_initialize(struct efa_qp *qp,
const struct ibv_qp_init_attr_ex *attr,
struct efa_create_qp_resp *resp)
{
struct efa_dev *dev = to_efa_dev(qp->verbs_qp.qp.context->device);
struct efa_context *ctx = to_efa_context(qp->verbs_qp.qp.context);
struct efa_wq_init_attr wq_attr;
struct efa_sq *sq = &qp->sq;
size_t desc_ring_size;
Expand Down Expand Up @@ -612,7 +610,7 @@ static int efa_sq_initialize(struct efa_qp *qp,
}

sq->desc += sq->desc_offset;
sq->max_wr_rdma_sge = min_t(uint16_t, dev->max_wr_rdma_sge,
sq->max_wr_rdma_sge = min_t(uint16_t, ctx->max_wr_rdma_sge,
EFA_IO_TX_DESC_NUM_RDMA_BUFS);

return 0;
Expand Down Expand Up @@ -738,7 +736,7 @@ static void efa_unlock_cqs(struct ibv_qp *ibvqp)
static void efa_qp_fill_wr_pfns(struct ibv_qp_ex *ibvqpx,
struct ibv_qp_init_attr_ex *attr_ex);

static int efa_check_qp_attr(struct efa_dev *dev,
static int efa_check_qp_attr(struct efa_context *ctx,
struct ibv_qp_init_attr_ex *attr,
struct efadv_qp_init_attr *efa_attr)
{
Expand All @@ -747,7 +745,7 @@ static int efa_check_qp_attr(struct efa_dev *dev,
IBV_QP_EX_WITH_SEND_WITH_IMM;
uint64_t supp_srd_send_ops_mask =
IBV_QP_EX_WITH_SEND | IBV_QP_EX_WITH_SEND_WITH_IMM |
(is_rdma_read_cap(dev) ? IBV_QP_EX_WITH_RDMA_READ : 0);
(is_rdma_read_cap(ctx) ? IBV_QP_EX_WITH_RDMA_READ : 0);

#define EFA_CREATE_QP_SUPP_ATTR_MASK \
(IBV_QP_INIT_ATTR_PD | IBV_QP_INIT_ATTR_SEND_OPS_FLAGS)
Expand Down Expand Up @@ -787,19 +785,19 @@ static int efa_check_qp_attr(struct efa_dev *dev,
return 0;
}

static int efa_check_qp_limits(struct efa_dev *dev,
static int efa_check_qp_limits(struct efa_context *ctx,
struct ibv_qp_init_attr_ex *attr)
{
if (attr->cap.max_send_sge > dev->max_sq_sge)
if (attr->cap.max_send_sge > ctx->max_sq_sge)
return EINVAL;

if (attr->cap.max_recv_sge > dev->max_rq_sge)
if (attr->cap.max_recv_sge > ctx->max_rq_sge)
return EINVAL;

if (attr->cap.max_send_wr > dev->max_sq_wr)
if (attr->cap.max_send_wr > ctx->max_sq_wr)
return EINVAL;

if (attr->cap.max_recv_wr > dev->max_rq_wr)
if (attr->cap.max_recv_wr > ctx->max_rq_wr)
return EINVAL;

return 0;
Expand All @@ -819,11 +817,11 @@ static struct ibv_qp *create_qp(struct ibv_context *ibvctx,
struct efa_qp *qp;
int err;

err = efa_check_qp_attr(dev, attr, efa_attr);
err = efa_check_qp_attr(ctx, attr, efa_attr);
if (err)
goto err_out;

err = efa_check_qp_limits(dev, attr);
err = efa_check_qp_limits(ctx, attr);
if (err)
goto err_out;

Expand Down

0 comments on commit d800c8a

Please sign in to comment.