mlx5: Introduce mlx5dv_wr_mr_interleaved post send builder

Introduce mlx5dv_wr_mr_interleaved() post send builder to be used for
issuing a WR that may register an interleaved memory layout.

Reviewed-by: Artemy Kovalyov <artemyko@mellanox.com>
Signed-off-by: Yishai Hadas <yishaih@mellanox.com>

providers/mlx5/man/mlx5dv_create_qp.3.md

@@ -38,6 +38,7 @@ struct mlx5dv_qp_init_attr {
 	uint64_t comp_mask;
 	uint32_t create_flags;
 	struct mlx5dv_dc_init_attr  dc_init_attr;
+	uint64_t send_ops_flags;
 };
 ```
 
@@ -47,6 +48,8 @@ struct mlx5dv_qp_init_attr {
 		valid values in *create_flags*
 	MLX5DV_QP_INIT_ATTR_MASK_DC:
 		valid values in *dc_init_attr*
+	MLX5DV_QP_INIT_ATTR_MASK_SEND_OPS_FLAGS:
+		valid values in *send_ops_flags*
 
 *create_flags*
 :	A bitwise OR of the various values described below.
@@ -95,6 +98,12 @@ struct mlx5dv_dc_init_attr {
 :	used to create a DCT QP.
 
 
+*send_ops_flags*
+:	A bitwise OR of the various values described below.
+
+	MLX5DV_QP_EX_WITH_MR_INTERLEAVED:
+		Enables the mlx5dv_wr_mr_interleaved() work requset on this QP.
+
 # NOTES
 
 **mlx5dv_qp_ex_from_ibv_qp_ex()** is used to get *struct mlx5dv_qp_ex* for

providers/mlx5/man/mlx5dv_wr_post.3.md

@@ -22,6 +22,20 @@ static inline void mlx5dv_wr_set_dc_addr(struct mlx5dv_qp_ex *mqp,
                                          struct ibv_ah *ah,
                                          uint32_t remote_dctn,
                                          uint64_t remote_dc_key);
+
+struct mlx5dv_mr_interleaved {
+	uint64_t        addr;
+	uint32_t        bytes_count;
+	uint32_t        bytes_skip;
+	uint32_t        lkey;
+};
+
+static inline void mlx5dv_wr_mr_interleaved(struct mlx5dv_qp_ex *mqp,
+					    struct mlx5dv_mkey *mkey,
+					    uint32_t access_flags, /* use enum ibv_access_flags */
+					    uint32_t repeat_count,
+					    uint16_t num_interleaved,
+					    struct mlx5dv_mr_interleaved *data);
 ```
 
 # DESCRIPTION
@@ -45,6 +59,27 @@ features on the posted WR.
 A work request creation requires to use the ibv_qp_ex as described in the
 man for ibv_wr_post and mlx5dv_qp with its available builders and setters.
 
+## QP Specific builders
+*RC* QPs
+:   *mlx5dv_wr_mr_interleaved()*
+
+    registers an interleaved memory layout by using an indirect mkey and some interleaved data.
+    The layout of the memory pointed by the mkey after its registration will be the *data* representation for the *num_interleaved* entries.
+    This single layout representation is repeated by *repeat_count*.
+
+    The *data* as described by struct mlx5dv_mr_interleaved will hold real data defined by *bytes_count* and then a padding of *bytes_skip*.
+    Post a successful registration, RDMA operations can use this *mkey*. The hardware will scatter the data according to the pattern.
+    The *mkey* should be used in a zero-based mode. The *addr* field in its *ibv_sge* is an offset in the total data.
+
+    Current implementation requires the IBV_SEND_INLINE option to be on in *ibv_qp_ex->wr_flags* field.
+    To be able to have more than 3 *num_interleaved* entries, the QP should be created with a larger WQE size that may fit it.
+    This should be done using the *max_inline_data* attribute of *struct ibv_qp_cap* upon its creation.
+
+    As one entry will be consumed for strided header, the *mkey* should be created with one more entry than the required *num_interleaved*.
+
+    In case *ibv_qp_ex->wr_flags* turns on IBV_SEND_SIGNALED, the reported WC opcode will be MLX5DV_WC_UMR.
+    Unregister the *mkey* to enable another pattern registration should be done via ibv_post_send with IBV_WR_LOCAL_INV opcode.
+
 ## QP Specific setters
 
 *DCI* QPs

providers/mlx5/mlx5dv.h

@@ -189,6 +189,7 @@ int mlx5dv_destroy_mkey(struct mlx5dv_mkey *mkey);
 enum mlx5dv_qp_init_attr_mask {
 	MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS	= 1 << 0,
 	MLX5DV_QP_INIT_ATTR_MASK_DC			= 1 << 1,
+	MLX5DV_QP_INIT_ATTR_MASK_SEND_OPS_FLAGS		= 1 << 2,
 };
 
 enum mlx5dv_dc_type {
@@ -201,16 +202,32 @@ struct mlx5dv_dc_init_attr {
 	uint64_t dct_access_key;
 };
 
+enum mlx5dv_qp_create_send_ops_flags {
+	MLX5DV_QP_EX_WITH_MR_INTERLEAVED	= 1 << 0,
+};
+
 struct mlx5dv_qp_init_attr {
 	uint64_t comp_mask;	/* Use enum mlx5dv_qp_init_attr_mask */
 	uint32_t create_flags;	/* Use enum mlx5dv_qp_create_flags */
 	struct mlx5dv_dc_init_attr  dc_init_attr;
+	uint64_t send_ops_flags; /* Use enum mlx5dv_qp_create_send_ops_flags */
 };
 
 struct ibv_qp *mlx5dv_create_qp(struct ibv_context *context,
 				struct ibv_qp_init_attr_ex *qp_attr,
 				struct mlx5dv_qp_init_attr *mlx5_qp_attr);
 
+struct mlx5dv_mr_interleaved {
+	uint64_t        addr;
+	uint32_t        bytes_count;
+	uint32_t        bytes_skip;
+	uint32_t        lkey;
+};
+
+enum mlx5dv_wc_opcode {
+	MLX5DV_WC_UMR = IBV_WC_DRIVER1,
+};
+
 struct mlx5dv_qp_ex {
 	uint64_t comp_mask;
 	/*
@@ -219,6 +236,12 @@ struct mlx5dv_qp_ex {
 	 */
 	void (*wr_set_dc_addr)(struct mlx5dv_qp_ex *mqp, struct ibv_ah *ah,
 			       uint32_t remote_dctn, uint64_t remote_dc_key);
+	void (*wr_mr_interleaved)(struct mlx5dv_qp_ex *mqp,
+				  struct mlx5dv_mkey *mkey,
+				  uint32_t access_flags, /* use enum ibv_access_flags */
+				  uint32_t repeat_count,
+				  uint16_t num_interleaved,
+				  struct mlx5dv_mr_interleaved *data);
 };
 
 struct mlx5dv_qp_ex *mlx5dv_qp_ex_from_ibv_qp_ex(struct ibv_qp_ex *qp);
@@ -231,6 +254,17 @@ static inline void mlx5dv_wr_set_dc_addr(struct mlx5dv_qp_ex *mqp,
 	mqp->wr_set_dc_addr(mqp, ah, remote_dctn, remote_dc_key);
 }
 
+static inline void mlx5dv_wr_mr_interleaved(struct mlx5dv_qp_ex *mqp,
+					    struct mlx5dv_mkey *mkey,
+					    uint32_t access_flags,
+					    uint32_t repeat_count,
+					    uint16_t num_interleaved,
+					    struct mlx5dv_mr_interleaved *data)
+{
+	mqp->wr_mr_interleaved(mqp, mkey, access_flags, repeat_count,
+			       num_interleaved, data);
+}
+
 enum mlx5dv_flow_action_esp_mask {
 	MLX5DV_FLOW_ACTION_ESP_MASK_FLAGS	= 1 << 0,
 };
@@ -843,6 +877,22 @@ union mlx5_wqe_umr_inline_seg {
 	struct mlx5_wqe_umr_klm_seg	klm;
 };
 
+struct mlx5_wqe_umr_repeat_ent_seg {
+	__be16		stride;
+	__be16		byte_count;
+	__be32		memkey;
+	__be64		va;
+};
+
+struct mlx5_wqe_umr_repeat_block_seg {
+	__be32		byte_count;
+	__be32		op;
+	__be32		repeat_count;
+	__be16		reserved;
+	__be16		num_ent;
+	struct mlx5_wqe_umr_repeat_ent_seg entries[0];
+};
+
 enum {
 	MLX5_WQE_MKEY_CONTEXT_FREE = 1 << 6
 };