Skip to content

Commit

Permalink
RDMA/mlx5: Move mlx5_ib_cont_pages() to the creation of the mlx5_ib_mr
Browse files Browse the repository at this point in the history 
For the user MR path, instead of calling this after getting the umem, call
it as part of creating the struct mlx5_ib_mr and distill its output to a
single page_shift stored inside the mr.

This avoids passing around the tuple of its output. Based on the umem and
page_shift, the output arguments can be computed using:

  count == ib_umem_num_pages(mr->umem)
  shift == mr->page_shift
  ncont == ib_umem_num_dma_blocks(mr->umem, 1 << mr->page_shift)
  order == order_base_2(ncont)

And since mr->page_shift == umem_odp->page_shift then ncont ==
ib_umem_num_dma_blocks() == ib_umem_odp_num_pages() for ODP umems.

Link: https://lore.kernel.org/r/20201026131936.1335664-5-leon@kernel.org
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
  • Loading branch information
@jgunthorpe
jgunthorpe committed Nov 2, 2020
1 parent 1c3d247 commit f0093fb
Show file tree
Hide file tree
Showing 3 changed files with 69 additions and 78 deletions.
11 changes: 11 additions & 0 deletions drivers/infiniband/hw/mlx5/mem.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,17 @@ void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr,
struct scatterlist *sg;
int entry;

if (umem->is_odp) {
struct ib_umem_odp *odp = to_ib_umem_odp(umem);

*shift = odp->page_shift;
*ncont = ib_umem_odp_num_pages(odp);
*count = *ncont << (*shift - PAGE_SHIFT);
if (order)
*order = ilog2(roundup_pow_of_two(*count));
return;
}

addr = addr >> PAGE_SHIFT;
tmp = (unsigned long)addr;
m = find_first_bit(&tmp, BITS_PER_LONG);
Expand Down
1 change: 1 addition & 0 deletions drivers/infiniband/hw/mlx5/mlx5_ib.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -597,6 +597,7 @@ struct mlx5_ib_mr {
int max_descs;
int desc_size;
int access_mode;
unsigned int page_shift;
struct mlx5_core_mkey mmkey;
struct ib_umem *umem;
struct mlx5_shared_mr_info *smr_info;
Expand Down
135 changes: 57 additions & 78 deletions drivers/infiniband/hw/mlx5/mr.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -868,14 +868,11 @@ static int mr_cache_max_order(struct mlx5_ib_dev *dev)
return MLX5_MAX_UMR_SHIFT;
}

static int mr_umem_get(struct mlx5_ib_dev *dev, u64 start, u64 length,
int access_flags, struct ib_umem **umem, int *npages,
int *page_shift, int *ncont, int *order)
static struct ib_umem *mr_umem_get(struct mlx5_ib_dev *dev, u64 start,
u64 length, int access_flags)
{
struct ib_umem *u;

*umem = NULL;

if (access_flags & IB_ACCESS_ON_DEMAND) {
struct ib_umem_odp *odp;

Expand All @@ -884,39 +881,17 @@ static int mr_umem_get(struct mlx5_ib_dev *dev, u64 start, u64 length,
if (IS_ERR(odp)) {
mlx5_ib_dbg(dev, "umem get failed (%ld)\n",
PTR_ERR(odp));
return PTR_ERR(odp);
}

u = &odp->umem;

*page_shift = odp->page_shift;
*ncont = ib_umem_odp_num_pages(odp);
*npages = *ncont << (*page_shift - PAGE_SHIFT);
if (order)
*order = ilog2(roundup_pow_of_two(*ncont));
} else {
u = ib_umem_get(&dev->ib_dev, start, length, access_flags);
if (IS_ERR(u)) {
mlx5_ib_dbg(dev, "umem get failed (%ld)\n", PTR_ERR(u));
return PTR_ERR(u);
return ERR_CAST(odp);
}

mlx5_ib_cont_pages(u, start, MLX5_MKEY_PAGE_SHIFT_MASK, npages,
page_shift, ncont, order);
return &odp->umem;
}

if (!*npages) {
mlx5_ib_warn(dev, "avoid zero region\n");
ib_umem_release(u);
return -EINVAL;
u = ib_umem_get(&dev->ib_dev, start, length, access_flags);
if (IS_ERR(u)) {
mlx5_ib_dbg(dev, "umem get failed (%ld)\n", PTR_ERR(u));
return u;
}

*umem = u;

mlx5_ib_dbg(dev, "npages %d, ncont %d, order %d, page_shift %d\n",
*npages, *ncont, *order, *page_shift);

return 0;
return u;
}

static void mlx5_ib_umr_done(struct ib_cq *cq, struct ib_wc *wc)
Expand Down Expand Up @@ -975,15 +950,21 @@ static struct mlx5_cache_ent *mr_cache_ent_from_order(struct mlx5_ib_dev *dev,
return &cache->ent[order];
}

static struct mlx5_ib_mr *
alloc_mr_from_cache(struct ib_pd *pd, struct ib_umem *umem, u64 virt_addr,
u64 len, int npages, int page_shift, unsigned int order,
int access_flags)
static struct mlx5_ib_mr *alloc_mr_from_cache(struct ib_pd *pd,
struct ib_umem *umem, u64 iova,
int access_flags)
{
struct mlx5_ib_dev *dev = to_mdev(pd->device);
struct mlx5_cache_ent *ent = mr_cache_ent_from_order(dev, order);
struct mlx5_cache_ent *ent;
struct mlx5_ib_mr *mr;
int npages;
int page_shift;
int ncont;
int order;

mlx5_ib_cont_pages(umem, iova, MLX5_MKEY_PAGE_SHIFT_MASK, &npages,
&page_shift, &ncont, &order);
ent = mr_cache_ent_from_order(dev, order);
if (!ent)
return ERR_PTR(-E2BIG);

Expand All @@ -1002,9 +983,10 @@ alloc_mr_from_cache(struct ib_pd *pd, struct ib_umem *umem, u64 virt_addr,
mr->umem = umem;
mr->access_flags = access_flags;
mr->desc_size = sizeof(struct mlx5_mtt);
mr->mmkey.iova = virt_addr;
mr->mmkey.size = len;
mr->mmkey.iova = iova;
mr->mmkey.size = umem->length;
mr->mmkey.pd = to_mpd(pd)->pdn;
mr->page_shift = page_shift;

return mr;
}
Expand Down Expand Up @@ -1163,14 +1145,15 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
* Else, the given ibmr will be used.
*/
static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,
u64 virt_addr, u64 length,
struct ib_umem *umem, int npages,
int page_shift, int access_flags,
bool populate)
struct ib_umem *umem, u64 iova,
int access_flags, bool populate)
{
struct mlx5_ib_dev *dev = to_mdev(pd->device);
struct mlx5_ib_mr *mr;
int page_shift;
__be64 *pas;
int npages;
int ncont;
void *mkc;
int inlen;
u32 *in;
Expand All @@ -1181,6 +1164,10 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,
if (!mr)
return ERR_PTR(-ENOMEM);

mlx5_ib_cont_pages(umem, iova, MLX5_MKEY_PAGE_SHIFT_MASK, &npages,
&page_shift, &ncont, NULL);

mr->page_shift = page_shift;
mr->ibmr.pd = pd;
mr->access_flags = access_flags;

Expand All @@ -1207,20 +1194,20 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,
MLX5_SET(create_mkey_in, in, pg_access, !!(pg_cap));

mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
set_mkc_access_pd_addr_fields(mkc, access_flags, virt_addr,
set_mkc_access_pd_addr_fields(mkc, access_flags, iova,
populate ? pd : dev->umrc.pd);
MLX5_SET(mkc, mkc, free, !populate);
MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
MLX5_SET(mkc, mkc, umr_en, 1);

MLX5_SET64(mkc, mkc, len, length);
MLX5_SET64(mkc, mkc, len, umem->length);
MLX5_SET(mkc, mkc, bsf_octword_size, 0);
MLX5_SET(mkc, mkc, translations_octword_size,
get_octo_len(virt_addr, length, page_shift));
get_octo_len(iova, umem->length, page_shift));
MLX5_SET(mkc, mkc, log_page_size, page_shift);
if (populate) {
MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
get_octo_len(virt_addr, length, page_shift));
get_octo_len(iova, umem->length, page_shift));
}

err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);
Expand Down Expand Up @@ -1358,10 +1345,6 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
struct mlx5_ib_mr *mr = NULL;
bool xlt_with_umr;
struct ib_umem *umem;
int page_shift;
int npages;
int ncont;
int order;
int err;

if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM))
Expand Down Expand Up @@ -1389,23 +1372,20 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
return &mr->ibmr;
}

err = mr_umem_get(dev, start, length, access_flags, &umem,
&npages, &page_shift, &ncont, &order);

if (err < 0)
return ERR_PTR(err);
umem = mr_umem_get(dev, start, length, access_flags);
if (IS_ERR(umem))
return ERR_CAST(umem);

if (xlt_with_umr) {
mr = alloc_mr_from_cache(pd, umem, virt_addr, length, ncont,
page_shift, order, access_flags);
mr = alloc_mr_from_cache(pd, umem, virt_addr, access_flags);
if (IS_ERR(mr))
mr = NULL;
}

if (!mr) {
mutex_lock(&dev->slow_path_mutex);
mr = reg_create(NULL, pd, virt_addr, length, umem, ncont,
page_shift, access_flags, !xlt_with_umr);
mr = reg_create(NULL, pd, umem, virt_addr, access_flags,
!xlt_with_umr);
mutex_unlock(&dev->slow_path_mutex);
}

Expand All @@ -1428,8 +1408,10 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
*/
int update_xlt_flags = MLX5_IB_UPD_XLT_ENABLE;

err = mlx5_ib_update_xlt(mr, 0, ncont, page_shift,
update_xlt_flags);
err = mlx5_ib_update_xlt(
mr, 0,
ib_umem_num_dma_blocks(umem, 1UL << mr->page_shift),
mr->page_shift, update_xlt_flags);
if (err) {
dereg_mr(dev, mr);
return ERR_PTR(err);
Expand Down Expand Up @@ -1519,11 +1501,7 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
int access_flags = flags & IB_MR_REREG_ACCESS ?
new_access_flags :
mr->access_flags;
int page_shift = 0;
int upd_flags = 0;
int npages = 0;
int ncont = 0;
int order = 0;
u64 addr, len;
int err;

Expand Down Expand Up @@ -1553,12 +1531,12 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
atomic_sub(ib_umem_num_pages(mr->umem),
&dev->mdev->priv.reg_pages);
ib_umem_release(mr->umem);
mr->umem = NULL;

err = mr_umem_get(dev, addr, len, access_flags, &mr->umem,
&npages, &page_shift, &ncont, &order);
if (err)
mr->umem = mr_umem_get(dev, addr, len, access_flags);
if (IS_ERR(mr->umem)) {
err = PTR_ERR(mr->umem);
mr->umem = NULL;
goto err;
}
atomic_add(ib_umem_num_pages(mr->umem),
&dev->mdev->priv.reg_pages);
}
Expand All @@ -1577,9 +1555,7 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
if (err)
goto err;

mr = reg_create(ib_mr, pd, addr, len, mr->umem, ncont,
page_shift, access_flags, true);

mr = reg_create(ib_mr, pd, mr->umem, addr, access_flags, true);
if (IS_ERR(mr)) {
err = PTR_ERR(mr);
mr = to_mmr(ib_mr);
Expand All @@ -1601,8 +1577,11 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
upd_flags |= MLX5_IB_UPD_XLT_PD;
if (flags & IB_MR_REREG_ACCESS)
upd_flags |= MLX5_IB_UPD_XLT_ACCESS;
err = mlx5_ib_update_xlt(mr, 0, npages, page_shift,
upd_flags);
err = mlx5_ib_update_xlt(
mr, 0,
ib_umem_num_dma_blocks(mr->umem,
1UL << mr->page_shift),
mr->page_shift, upd_flags);
} else {
err = rereg_umr(pd, mr, access_flags, flags);
}
Expand Down

0 comments on commit f0093fb

Please sign in to comment.