verbs: Add interfaces to configure and use ECE

ECE parameters are vendor specific information per-QP,
provide a way to set and query ECE data.

 * ibv_set_ece() - overwrite default ECE options and instruct
	libibverbs to use ECE data while enabling QP.
 * ibv_query_ece() - get ECE options.

Signed-off-by: Leon Romanovsky <leonro@mellanox.com>

debian/libibverbs1.symbols

@@ -7,6 +7,7 @@ libibverbs.so.1 libibverbs1 #MINVER#
  IBVERBS_1.7@IBVERBS_1.7 25
  IBVERBS_1.8@IBVERBS_1.8 28
  IBVERBS_1.9@IBVERBS_1.9 30
+ IBVERBS_1.10@IBVERBS_1.10 31
  (symver)IBVERBS_PRIVATE_25 25
  ibv_ack_async_event@IBVERBS_1.0 1.1.6
  ibv_ack_async_event@IBVERBS_1.1 1.1.6
@@ -78,6 +79,7 @@ libibverbs.so.1 libibverbs1 #MINVER#
  ibv_qp_to_qp_ex@IBVERBS_1.6 24
  ibv_query_device@IBVERBS_1.0 1.1.6
  ibv_query_device@IBVERBS_1.1 1.1.6
+ ibv_query_ece@IBVERBS_1.10 31
  ibv_query_gid@IBVERBS_1.0 1.1.6
  ibv_query_gid@IBVERBS_1.1 1.1.6
  ibv_query_pkey@IBVERBS_1.0 1.1.6
@@ -100,6 +102,7 @@ libibverbs.so.1 libibverbs1 #MINVER#
  ibv_resize_cq@IBVERBS_1.0 1.1.6
  ibv_resize_cq@IBVERBS_1.1 1.1.6
  ibv_resolve_eth_l2_from_gid@IBVERBS_1.1 1.2.0
+ ibv_set_ece@IBVERBS_1.10 31
  ibv_wc_status_str@IBVERBS_1.1 1.1.6
  mbps_to_ibv_rate@IBVERBS_1.1 1.1.8
  mult_to_ibv_rate@IBVERBS_1.0 1.1.6

libibverbs/CMakeLists.txt

@@ -21,7 +21,7 @@ configure_file("libibverbs.map.in"
 
 rdma_library(ibverbs "${CMAKE_CURRENT_BINARY_DIR}/libibverbs.map"
   # See Documentation/versioning.md
-  1 1.9.${PACKAGE_VERSION}
+  1 1.10.${PACKAGE_VERSION}
   all_providers.c
   cmd.c
   cmd_ah.c

libibverbs/driver.h

@@ -339,6 +339,7 @@ struct verbs_context_ops {
 			       const struct ibv_query_device_ex_input *input,
 			       struct ibv_device_attr_ex *attr,
 			       size_t attr_size);
+	int (*query_ece)(struct ibv_qp *qp, struct ibv_ece *ece);
 	int (*query_port)(struct ibv_context *context, uint8_t port_num,
 			  struct ibv_port_attr *port_attr);
 	int (*query_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr,
@@ -359,6 +360,7 @@ struct verbs_context_ops {
 	int (*rereg_mr)(struct verbs_mr *vmr, int flags, struct ibv_pd *pd,
 			void *addr, size_t length, int access);
 	int (*resize_cq)(struct ibv_cq *cq, int cqe);
+	int (*set_ece)(struct ibv_qp *qp, struct ibv_ece *ece);
 };
 
 static inline struct verbs_device *

libibverbs/dummy_ops.c

@@ -387,6 +387,11 @@ static int query_device_ex(struct ibv_context *context,
 	return ibv_query_device(context, &attr->orig_attr);
 }
 
+static int query_ece(struct ibv_qp *qp, struct ibv_ece *ece)
+{
+	return EOPNOTSUPP;
+}
+
 static int query_port(struct ibv_context *context, uint8_t port_num,
 		      struct ibv_port_attr *port_attr)
 {
@@ -450,6 +455,11 @@ static int resize_cq(struct ibv_cq *cq, int cqe)
 	return EOPNOTSUPP;
 }
 
+static int set_ece(struct ibv_qp *qp, struct ibv_ece *ece)
+{
+	return EOPNOTSUPP;
+}
+
 /*
  * Ops in verbs_dummy_ops simply return an EOPNOTSUPP error code when called, or
  * do nothing. They are placed in the ops structures if the provider does not
@@ -519,6 +529,7 @@ const struct verbs_context_ops verbs_dummy_ops = {
 	post_srq_recv,
 	query_device,
 	query_device_ex,
+	query_ece,
 	query_port,
 	query_qp,
 	query_rt_values,
@@ -529,6 +540,7 @@ const struct verbs_context_ops verbs_dummy_ops = {
 	req_notify_cq,
 	rereg_mr,
 	resize_cq,
+	set_ece,
 };
 
 /*
@@ -635,6 +647,7 @@ void verbs_set_ops(struct verbs_context *vctx,
 	SET_OP(ctx, post_srq_recv);
 	SET_PRIV_OP(ctx, query_device);
 	SET_OP(vctx, query_device_ex);
+	SET_PRIV_OP_IC(vctx, query_ece);
 	SET_PRIV_OP_IC(ctx, query_port);
 	SET_PRIV_OP(ctx, query_qp);
 	SET_OP(vctx, query_rt_values);
@@ -645,6 +658,7 @@ void verbs_set_ops(struct verbs_context *vctx,
 	SET_OP(ctx, req_notify_cq);
 	SET_PRIV_OP(ctx, rereg_mr);
 	SET_PRIV_OP(ctx, resize_cq);
+	SET_PRIV_OP_IC(vctx, set_ece);
 
 #undef SET_OP
 #undef SET_OP2

libibverbs/libibverbs.map.in

@@ -131,6 +131,12 @@ IBVERBS_1.9 {
 		ibv_get_device_index;
 } IBVERBS_1.8;
 
+IBVERBS_1.10 {
+	global:
+		ibv_query_ece;
+		ibv_set_ece;
+} IBVERBS_1.9;
+
 /* If any symbols in this stanza change ABI then the entire staza gets a new symbol
    version. See the top level CMakeLists.txt for this setting. */
 

libibverbs/man/CMakeLists.txt

@@ -52,6 +52,7 @@ rdma_man_pages(
   ibv_post_srq_recv.3
   ibv_query_device.3
   ibv_query_device_ex.3
+  ibv_query_ece.3.md
   ibv_query_gid.3.md
   ibv_query_pkey.3.md
   ibv_query_port.3
@@ -66,6 +67,7 @@ rdma_man_pages(
   ibv_req_notify_cq.3.md
   ibv_rereg_mr.3.md
   ibv_resize_cq.3.md
+  ibv_set_ece.3.md
   ibv_srq_pingpong.1
   ibv_uc_pingpong.1
   ibv_ud_pingpong.1

libibverbs/man/ibv_query_ece.3.md

@@ -0,0 +1,75 @@
+---
+date: 2020-01-22
+footer: libibverbs
+header: "Libibverbs Programmer's Manual"
+layout: page
+license: 'Licensed under the OpenIB.org BSD license (FreeBSD Variant) - See COPYING.md'
+section: 3
+title: IBV_QUERY_ECE
+---
+
+# NAME
+
+ibv_query_ece - query ECE options.
+
+# SYNOPSIS
+
+```c
+#include <infiniband/verbs.h>
+
+int ibv_query_ece(struct ibv_qp *qp, struct ibv_ece *ece);
+```
+
+# DESCRIPTION
+
+**ibv_query_ece()** query ECE options.
+
+Return to the user current ECE state for the QP.
+
+# ARGUMENTS
+*qp*
+:	The queue pair (QP) associated with the ECE options.
+
+## *ece* Argument
+:	The ECE values.
+
+```c
+struct ibv_ece {
+	uint32_t vendor_id;
+	uint32_t options;
+	uint32_t comp_mask;
+};
+```
+
+*vendor_id*
+:	Unique identifier of the provider vendor on the network.
+	The providers will set IEEE OUI here to distinguish itself
+	in non-homogenius network.
+
+*options*
+:	Provider specific attributes which are supported.
+
+*comp_mask*
+:	Bitmask specifying what fields in the structure are valid.
+
+# RETURN VALUE
+
+**ibv_query_ece()** returns 0 when the call was successful, or the errno value
+		    which indicates the failure reason.
+
+*EOPNOTSUPP*
+:	libibverbs or provider driver doesn't support the ibv_set_ece() verb.
+
+*EINVAL*
+:	In one of the following:
+	o The QP is invalid.
+	o The ECE options are invalid.
+
+# SEE ALSO
+
+**ibv_set_ece**(3),
+
+# AUTHOR
+
+Leon Romanovsky <leonro@mellanox.com>
+

libibverbs/man/ibv_set_ece.3.md

@@ -0,0 +1,78 @@
+---
+date: 2020-01-22
+footer: libibverbs
+header: "Libibverbs Programmer's Manual"
+layout: page
+license: 'Licensed under the OpenIB.org BSD license (FreeBSD Variant) - See COPYING.md'
+section: 3
+title: IBV_SET_ECE
+---
+
+# NAME
+
+ibv_set_ece - set ECE options and use them for QP configuration stage.
+
+# SYNOPSIS
+
+```c
+#include <infiniband/verbs.h>
+
+int ibv_set_ece(struct ibv_qp *qp, struct ibv_ece *ece);
+```
+
+# DESCRIPTION
+
+**ibv_set_ece()** set ECE options and use them for QP configuration stage.
+
+The desired ECE options will be used during various modify QP stages
+based on supported options in relevant QP state.
+
+# ARGUMENTS
+*qp*
+:	The queue pair (QP) associated with the ECE options.
+
+## *ece* Argument
+:	The requested ECE values. This is IN/OUT field, the accepted options
+        will be returned in this field.
+
+```c
+struct ibv_ece {
+	uint32_t vendor_id;
+	uint32_t options;
+	uint32_t comp_mask;
+};
+```
+
+*vendor_id*
+:	Unique identifier of the provider vendor on the network.
+	The providers will set IEEE OUI here to distinguish itself
+	in non-homogenius network.
+
+*options*
+:	Provider specific attributes which are supported or
+	needed to be enabled by ECE users.
+
+*comp_mask*
+:	Bitmask specifying what fields in the structure are valid.
+
+# RETURN VALUE
+
+**ibv_set_ece()** returns 0 when the call was successful, or the errno value
+	which indicates the failure reason.
+
+*EOPNOTSUPP*
+:	libibverbs or provider driver doesn't support the ibv_set_ece() verb.
+
+*EINVAL*
+:	In one of the following:
+	o The QP is invalid.
+	o The ECE options are invalid.
+
+# SEE ALSO
+
+**ibv_query_ece**(3),
+
+# AUTHOR
+
+Leon Romanovsky <leonro@mellanox.com>
+

libibverbs/verbs.c

@@ -1066,3 +1066,18 @@ int ibv_resolve_eth_l2_from_gid(struct ibv_context *context,
 
 	return ret;
 }
+
+int ibv_set_ece(struct ibv_qp *qp, struct ibv_ece *ece)
+{
+	if (!ece->vendor_id) {
+		errno = EOPNOTSUPP;
+		return errno;
+	}
+
+	return get_ops(qp->context)->set_ece(qp, ece);
+}
+
+int ibv_query_ece(struct ibv_qp *qp, struct ibv_ece *ece)
+{
+	return get_ops(qp->context)->query_ece(qp, ece);
+}

libibverbs/verbs.h

@@ -1401,6 +1401,21 @@ static inline void ibv_wr_abort(struct ibv_qp_ex *qp)
 	qp->wr_abort(qp);
 }
 
+struct ibv_ece {
+	/*
+	 * Unique identifier of the provider vendor on the network.
+	 * The providers will set IEEE OUI here to distinguish
+	 * itself in non-homogenius network.
+	 */
+	uint32_t vendor_id;
+	/*
+	 * Provider specific attributes which are supported or
+	 * needed to be enabled by ECE users.
+	 */
+	uint32_t options;
+	uint32_t comp_mask;
+};
+
 struct ibv_comp_channel {
 	struct ibv_context     *context;
 	int			fd;
@@ -3352,6 +3367,15 @@ static inline uint16_t ibv_flow_label_to_udp_sport(uint32_t fl)
 	return (uint16_t)(fl_low | IB_ROCE_UDP_ENCAP_VALID_PORT_MIN);
 }
 
+/**
+ * ibv_set_ece - Set ECE options
+ */
+int ibv_set_ece(struct ibv_qp *qp, struct ibv_ece *ece);
+
+/**
+ * ibv_query_ece - Get accepted ECE options
+ */
+int ibv_query_ece(struct ibv_qp *qp, struct ibv_ece *ece);
 #ifdef __cplusplus
 }
 #endif