From 55b51135cab1c0a7f28516a636d5790c2cfca225 Mon Sep 17 00:00:00 2001 From: Leon Romanovsky Date: Sun, 19 Jan 2020 19:03:30 +0200 Subject: [PATCH] verbs: Add interfaces to configure and use ECE ECE parameters are vendor specific information per-QP, provide a way to set and query ECE data. * ibv_set_ece() - overwrite default ECE options and instruct libibverbs to use ECE data while enabling QP. * ibv_query_ece() - get ECE options. Signed-off-by: Leon Romanovsky --- debian/libibverbs1.symbols | 3 ++ libibverbs/CMakeLists.txt | 2 +- libibverbs/driver.h | 2 + libibverbs/dummy_ops.c | 14 ++++++ libibverbs/libibverbs.map.in | 6 +++ libibverbs/man/CMakeLists.txt | 2 + libibverbs/man/ibv_query_ece.3.md | 75 +++++++++++++++++++++++++++++ libibverbs/man/ibv_set_ece.3.md | 78 +++++++++++++++++++++++++++++++ libibverbs/verbs.c | 15 ++++++ libibverbs/verbs.h | 24 ++++++++++ 10 files changed, 220 insertions(+), 1 deletion(-) create mode 100644 libibverbs/man/ibv_query_ece.3.md create mode 100644 libibverbs/man/ibv_set_ece.3.md diff --git a/debian/libibverbs1.symbols b/debian/libibverbs1.symbols index 10ca9bf3e..bb7eac952 100644 --- a/debian/libibverbs1.symbols +++ b/debian/libibverbs1.symbols @@ -7,6 +7,7 @@ libibverbs.so.1 libibverbs1 #MINVER# IBVERBS_1.7@IBVERBS_1.7 25 IBVERBS_1.8@IBVERBS_1.8 28 IBVERBS_1.9@IBVERBS_1.9 30 + IBVERBS_1.10@IBVERBS_1.10 31 (symver)IBVERBS_PRIVATE_25 25 ibv_ack_async_event@IBVERBS_1.0 1.1.6 ibv_ack_async_event@IBVERBS_1.1 1.1.6 @@ -78,6 +79,7 @@ libibverbs.so.1 libibverbs1 #MINVER# ibv_qp_to_qp_ex@IBVERBS_1.6 24 ibv_query_device@IBVERBS_1.0 1.1.6 ibv_query_device@IBVERBS_1.1 1.1.6 + ibv_query_ece@IBVERBS_1.10 31 ibv_query_gid@IBVERBS_1.0 1.1.6 ibv_query_gid@IBVERBS_1.1 1.1.6 ibv_query_pkey@IBVERBS_1.0 1.1.6 @@ -100,6 +102,7 @@ libibverbs.so.1 libibverbs1 #MINVER# ibv_resize_cq@IBVERBS_1.0 1.1.6 ibv_resize_cq@IBVERBS_1.1 1.1.6 ibv_resolve_eth_l2_from_gid@IBVERBS_1.1 1.2.0 + ibv_set_ece@IBVERBS_1.10 31 ibv_wc_status_str@IBVERBS_1.1 1.1.6 mbps_to_ibv_rate@IBVERBS_1.1 1.1.8 mult_to_ibv_rate@IBVERBS_1.0 1.1.6 diff --git a/libibverbs/CMakeLists.txt b/libibverbs/CMakeLists.txt index 7e4668ede..06a590fc3 100644 --- a/libibverbs/CMakeLists.txt +++ b/libibverbs/CMakeLists.txt @@ -21,7 +21,7 @@ configure_file("libibverbs.map.in" rdma_library(ibverbs "${CMAKE_CURRENT_BINARY_DIR}/libibverbs.map" # See Documentation/versioning.md - 1 1.9.${PACKAGE_VERSION} + 1 1.10.${PACKAGE_VERSION} all_providers.c cmd.c cmd_ah.c diff --git a/libibverbs/driver.h b/libibverbs/driver.h index de81955db..5eb70b38c 100644 --- a/libibverbs/driver.h +++ b/libibverbs/driver.h @@ -339,6 +339,7 @@ struct verbs_context_ops { const struct ibv_query_device_ex_input *input, struct ibv_device_attr_ex *attr, size_t attr_size); + int (*query_ece)(struct ibv_qp *qp, struct ibv_ece *ece); int (*query_port)(struct ibv_context *context, uint8_t port_num, struct ibv_port_attr *port_attr); int (*query_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr, @@ -359,6 +360,7 @@ struct verbs_context_ops { int (*rereg_mr)(struct verbs_mr *vmr, int flags, struct ibv_pd *pd, void *addr, size_t length, int access); int (*resize_cq)(struct ibv_cq *cq, int cqe); + int (*set_ece)(struct ibv_qp *qp, struct ibv_ece *ece); }; static inline struct verbs_device * diff --git a/libibverbs/dummy_ops.c b/libibverbs/dummy_ops.c index 32fec71f0..37cd11005 100644 --- a/libibverbs/dummy_ops.c +++ b/libibverbs/dummy_ops.c @@ -387,6 +387,11 @@ static int query_device_ex(struct ibv_context *context, return ibv_query_device(context, &attr->orig_attr); } +static int query_ece(struct ibv_qp *qp, struct ibv_ece *ece) +{ + return EOPNOTSUPP; +} + static int query_port(struct ibv_context *context, uint8_t port_num, struct ibv_port_attr *port_attr) { @@ -450,6 +455,11 @@ static int resize_cq(struct ibv_cq *cq, int cqe) return EOPNOTSUPP; } +static int set_ece(struct ibv_qp *qp, struct ibv_ece *ece) +{ + return EOPNOTSUPP; +} + /* * Ops in verbs_dummy_ops simply return an EOPNOTSUPP error code when called, or * do nothing. They are placed in the ops structures if the provider does not @@ -519,6 +529,7 @@ const struct verbs_context_ops verbs_dummy_ops = { post_srq_recv, query_device, query_device_ex, + query_ece, query_port, query_qp, query_rt_values, @@ -529,6 +540,7 @@ const struct verbs_context_ops verbs_dummy_ops = { req_notify_cq, rereg_mr, resize_cq, + set_ece, }; /* @@ -635,6 +647,7 @@ void verbs_set_ops(struct verbs_context *vctx, SET_OP(ctx, post_srq_recv); SET_PRIV_OP(ctx, query_device); SET_OP(vctx, query_device_ex); + SET_PRIV_OP_IC(vctx, query_ece); SET_PRIV_OP_IC(ctx, query_port); SET_PRIV_OP(ctx, query_qp); SET_OP(vctx, query_rt_values); @@ -645,6 +658,7 @@ void verbs_set_ops(struct verbs_context *vctx, SET_OP(ctx, req_notify_cq); SET_PRIV_OP(ctx, rereg_mr); SET_PRIV_OP(ctx, resize_cq); + SET_PRIV_OP_IC(vctx, set_ece); #undef SET_OP #undef SET_OP2 diff --git a/libibverbs/libibverbs.map.in b/libibverbs/libibverbs.map.in index f0d79c78f..61b6a8038 100644 --- a/libibverbs/libibverbs.map.in +++ b/libibverbs/libibverbs.map.in @@ -131,6 +131,12 @@ IBVERBS_1.9 { ibv_get_device_index; } IBVERBS_1.8; +IBVERBS_1.10 { + global: + ibv_query_ece; + ibv_set_ece; +} IBVERBS_1.9; + /* If any symbols in this stanza change ABI then the entire staza gets a new symbol version. See the top level CMakeLists.txt for this setting. */ diff --git a/libibverbs/man/CMakeLists.txt b/libibverbs/man/CMakeLists.txt index 87f00185b..a9cf61e31 100644 --- a/libibverbs/man/CMakeLists.txt +++ b/libibverbs/man/CMakeLists.txt @@ -52,6 +52,7 @@ rdma_man_pages( ibv_post_srq_recv.3 ibv_query_device.3 ibv_query_device_ex.3 + ibv_query_ece.3.md ibv_query_gid.3.md ibv_query_pkey.3.md ibv_query_port.3 @@ -66,6 +67,7 @@ rdma_man_pages( ibv_req_notify_cq.3.md ibv_rereg_mr.3.md ibv_resize_cq.3.md + ibv_set_ece.3.md ibv_srq_pingpong.1 ibv_uc_pingpong.1 ibv_ud_pingpong.1 diff --git a/libibverbs/man/ibv_query_ece.3.md b/libibverbs/man/ibv_query_ece.3.md new file mode 100644 index 000000000..9b6f19d82 --- /dev/null +++ b/libibverbs/man/ibv_query_ece.3.md @@ -0,0 +1,75 @@ +--- +date: 2020-01-22 +footer: libibverbs +header: "Libibverbs Programmer's Manual" +layout: page +license: 'Licensed under the OpenIB.org BSD license (FreeBSD Variant) - See COPYING.md' +section: 3 +title: IBV_QUERY_ECE +--- + +# NAME + +ibv_query_ece - query ECE options. + +# SYNOPSIS + +```c +#include + +int ibv_query_ece(struct ibv_qp *qp, struct ibv_ece *ece); +``` + +# DESCRIPTION + +**ibv_query_ece()** query ECE options. + +Return to the user current ECE state for the QP. + +# ARGUMENTS +*qp* +: The queue pair (QP) associated with the ECE options. + +## *ece* Argument +: The ECE values. + +```c +struct ibv_ece { + uint32_t vendor_id; + uint32_t options; + uint32_t comp_mask; +}; +``` + +*vendor_id* +: Unique identifier of the provider vendor on the network. + The providers will set IEEE OUI here to distinguish itself + in non-homogenius network. + +*options* +: Provider specific attributes which are supported. + +*comp_mask* +: Bitmask specifying what fields in the structure are valid. + +# RETURN VALUE + +**ibv_query_ece()** returns 0 when the call was successful, or the errno value + which indicates the failure reason. + +*EOPNOTSUPP* +: libibverbs or provider driver doesn't support the ibv_set_ece() verb. + +*EINVAL* +: In one of the following: + o The QP is invalid. + o The ECE options are invalid. + +# SEE ALSO + +**ibv_set_ece**(3), + +# AUTHOR + +Leon Romanovsky + diff --git a/libibverbs/man/ibv_set_ece.3.md b/libibverbs/man/ibv_set_ece.3.md new file mode 100644 index 000000000..d7910c895 --- /dev/null +++ b/libibverbs/man/ibv_set_ece.3.md @@ -0,0 +1,78 @@ +--- +date: 2020-01-22 +footer: libibverbs +header: "Libibverbs Programmer's Manual" +layout: page +license: 'Licensed under the OpenIB.org BSD license (FreeBSD Variant) - See COPYING.md' +section: 3 +title: IBV_SET_ECE +--- + +# NAME + +ibv_set_ece - set ECE options and use them for QP configuration stage. + +# SYNOPSIS + +```c +#include + +int ibv_set_ece(struct ibv_qp *qp, struct ibv_ece *ece); +``` + +# DESCRIPTION + +**ibv_set_ece()** set ECE options and use them for QP configuration stage. + +The desired ECE options will be used during various modify QP stages +based on supported options in relevant QP state. + +# ARGUMENTS +*qp* +: The queue pair (QP) associated with the ECE options. + +## *ece* Argument +: The requested ECE values. This is IN/OUT field, the accepted options + will be returned in this field. + +```c +struct ibv_ece { + uint32_t vendor_id; + uint32_t options; + uint32_t comp_mask; +}; +``` + +*vendor_id* +: Unique identifier of the provider vendor on the network. + The providers will set IEEE OUI here to distinguish itself + in non-homogenius network. + +*options* +: Provider specific attributes which are supported or + needed to be enabled by ECE users. + +*comp_mask* +: Bitmask specifying what fields in the structure are valid. + +# RETURN VALUE + +**ibv_set_ece()** returns 0 when the call was successful, or the errno value + which indicates the failure reason. + +*EOPNOTSUPP* +: libibverbs or provider driver doesn't support the ibv_set_ece() verb. + +*EINVAL* +: In one of the following: + o The QP is invalid. + o The ECE options are invalid. + +# SEE ALSO + +**ibv_query_ece**(3), + +# AUTHOR + +Leon Romanovsky + diff --git a/libibverbs/verbs.c b/libibverbs/verbs.c index f38003681..77a7a2533 100644 --- a/libibverbs/verbs.c +++ b/libibverbs/verbs.c @@ -1066,3 +1066,18 @@ int ibv_resolve_eth_l2_from_gid(struct ibv_context *context, return ret; } + +int ibv_set_ece(struct ibv_qp *qp, struct ibv_ece *ece) +{ + if (!ece->vendor_id) { + errno = EOPNOTSUPP; + return errno; + } + + return get_ops(qp->context)->set_ece(qp, ece); +} + +int ibv_query_ece(struct ibv_qp *qp, struct ibv_ece *ece) +{ + return get_ops(qp->context)->query_ece(qp, ece); +} diff --git a/libibverbs/verbs.h b/libibverbs/verbs.h index 18bc9b041..f72c1ab30 100644 --- a/libibverbs/verbs.h +++ b/libibverbs/verbs.h @@ -1401,6 +1401,21 @@ static inline void ibv_wr_abort(struct ibv_qp_ex *qp) qp->wr_abort(qp); } +struct ibv_ece { + /* + * Unique identifier of the provider vendor on the network. + * The providers will set IEEE OUI here to distinguish + * itself in non-homogenius network. + */ + uint32_t vendor_id; + /* + * Provider specific attributes which are supported or + * needed to be enabled by ECE users. + */ + uint32_t options; + uint32_t comp_mask; +}; + struct ibv_comp_channel { struct ibv_context *context; int fd; @@ -3352,6 +3367,15 @@ static inline uint16_t ibv_flow_label_to_udp_sport(uint32_t fl) return (uint16_t)(fl_low | IB_ROCE_UDP_ENCAP_VALID_PORT_MIN); } +/** + * ibv_set_ece - Set ECE options + */ +int ibv_set_ece(struct ibv_qp *qp, struct ibv_ece *ece); + +/** + * ibv_query_ece - Get accepted ECE options + */ +int ibv_query_ece(struct ibv_qp *qp, struct ibv_ece *ece); #ifdef __cplusplus } #endif