From cf053bc6a58db29e781b4b7288ca964aca88b745 Mon Sep 17 00:00:00 2001 From: Ido Kalir Date: Thu, 12 Mar 2020 12:53:19 +0200 Subject: [PATCH] pyverbs: Add support for ECE ECE (enhanced connection establishment) is a mechanism that gives an option to different libibverbs providers to advertise and use various provider-specific QP configuration options. Add those verbs: ibv_query_ece - get QPs ece. ibv_set_ece - set the QPs ece. rdma_set_local_ece - set the local CMs ece. rdma_get_remote_ece - get the remote CM ece from the connection request. Signed-off-by: Maxim Chicherin Signed-off-by: Ido Kalir --- pyverbs/cmid.pyx | 23 +++++++++++++++++++- pyverbs/libibverbs.pxd | 7 ++++++ pyverbs/librdmacm.pxd | 2 ++ pyverbs/qp.pxd | 3 +++ pyverbs/qp.pyx | 49 ++++++++++++++++++++++++++++++++++++++++-- 5 files changed, 81 insertions(+), 3 deletions(-) diff --git a/pyverbs/cmid.pyx b/pyverbs/cmid.pyx index cf8a9d2f4..5488f04e0 100755 --- a/pyverbs/cmid.pyx +++ b/pyverbs/cmid.pyx @@ -2,7 +2,7 @@ from libc.stdint cimport uintptr_t from libc.string cimport memset from pyverbs.pyverbs_error import PyverbsUserError -from pyverbs.qp cimport QPInitAttr, QPAttr +from pyverbs.qp cimport QPInitAttr, QPAttr, ECE from pyverbs.base import PyverbsRDMAErrno cimport pyverbs.libibverbs_enums as e cimport pyverbs.librdmacm_enums as ce @@ -536,6 +536,27 @@ cdef class CMID(PyverbsCM): if ret != 0: raise PyverbsRDMAErrno('Failed to Complete an active connection request') + def set_local_ece(self, ECE ece): + """ + Set local ECE paraemters to be used for REQ/REP communication. + :param ece: ECE object with the requested configuration + :return: None + """ + rc = cm.rdma_set_local_ece(self.id, &ece.ece) + if rc != 0: + raise PyverbsRDMAErrno('Failed to set local ECE') + + def get_remote_ece(self): + """ + Get ECE parameters as were received from the communication peer. + :return: ECE object with the ece configuration + """ + ece = ECE() + rc = cm.rdma_get_remote_ece(self.id, &ece.ece) + if rc != 0: + raise PyverbsRDMAErrno('Failed to get remote ECE') + return ece + def create_qp(self, QPInitAttr qp_init not None): """ Create a QP, which is associated with CMID. diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index 4beb434fb..7a75511f1 100755 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -475,6 +475,11 @@ cdef extern from 'infiniband/verbs.h': uint64_t wr_id unsigned int wr_flags + cdef struct ibv_ece: + uint32_t vendor_id + uint32_t options + uint32_t comp_mask + ibv_device **ibv_get_device_list(int *n) void ibv_free_device_list(ibv_device **list) ibv_context *ibv_open_device(ibv_device *device) @@ -601,3 +606,5 @@ cdef extern from 'infiniband/verbs.h': cdef extern from 'infiniband/driver.h': int ibv_query_gid_type(ibv_context *context, uint8_t port_num, unsigned int index, ibv_gid_type *type) + int ibv_set_ece(ibv_qp *qp, ibv_ece *ece) + int ibv_query_ece(ibv_qp *qp, ibv_ece *ece) diff --git a/pyverbs/librdmacm.pxd b/pyverbs/librdmacm.pxd index 645657b80..170d728ec 100755 --- a/pyverbs/librdmacm.pxd +++ b/pyverbs/librdmacm.pxd @@ -102,6 +102,8 @@ cdef extern from '': int rdma_create_id(rdma_event_channel *channel, rdma_cm_id **id, void *context, rdma_port_space ps) int rdma_destroy_id(rdma_cm_id *id) + int rdma_get_remote_ece(rdma_cm_id *id, ibv_ece *ece) + int rdma_set_local_ece(rdma_cm_id *id, ibv_ece *ece) int rdma_get_request(rdma_cm_id *listen, rdma_cm_id **id) int rdma_bind_addr(rdma_cm_id *id, sockaddr *addr) int rdma_resolve_addr(rdma_cm_id *id, sockaddr *src_addr, diff --git a/pyverbs/qp.pxd b/pyverbs/qp.pxd index f891e27e8..44adfe9b1 100644 --- a/pyverbs/qp.pxd +++ b/pyverbs/qp.pxd @@ -45,3 +45,6 @@ cdef class DataBuffer(PyverbsCM): cdef class QPEx(QP): cdef v.ibv_qp_ex *qp_ex + +cdef class ECE(PyverbsCM): + cdef v.ibv_ece ece diff --git a/pyverbs/qp.pyx b/pyverbs/qp.pyx index 20a45b036..d8848add1 100755 --- a/pyverbs/qp.pyx +++ b/pyverbs/qp.pyx @@ -5,9 +5,8 @@ from libc.stdlib cimport malloc, free from libc.string cimport memcpy import weakref +from pyverbs.pyverbs_error import PyverbsUserError, PyverbsError, PyverbsRDMAError from pyverbs.utils import gid_str, qp_type_to_str, qp_state_to_str, mtu_to_str -from pyverbs.pyverbs_error import PyverbsUserError, PyverbsError, \ - PyverbsRDMAError from pyverbs.utils import access_flags_to_str, mig_state_to_str from pyverbs.base import PyverbsRDMAErrno from pyverbs.wr cimport RecvWR, SendWR, SGE @@ -873,6 +872,28 @@ cdef class QPAttr(PyverbsObject): print_format.format('Rate limit', self.attr.rate_limit) +cdef class ECE(PyverbsCM): + def __init__(self, vendor_id=0, options=0, comp_mask=0): + """ + :param vendor_id: Unique identifier of the provider vendor. + :param options: Provider specific attributes which are supported or + needed to be enabled by ECE users. + :param comp_mask: A bitmask specifying which ECE options should be + valid. + """ + super().__init__() + self.ece.vendor_id = vendor_id + self.ece.options = options + self.ece.comp_mask = comp_mask + + def __str__(self): + print_format = '{:22}: 0x{:<20x}\n' + return 'ECE:\n' +\ + print_format.format('Vendor ID', self.ece.vendor_id) +\ + print_format.format('Options', self.ece.options) +\ + print_format.format('Comp Mask', self.ece.comp_mask) + + cdef class QP(PyverbsCM): def __init__(self, object creator not None, object init_attr not None, QPAttr qp_attr=None): @@ -1143,6 +1164,30 @@ cdef class QP(PyverbsCM): memcpy(&bad_wr.send_wr, my_bad_wr, sizeof(bad_wr.send_wr)) raise PyverbsRDMAError('Failed to post send', rc) + def set_ece(self, ECE ece): + """ + Set ECE options and use them for QP configuration stage + :param ece: The requested ECE values. + :return: None + """ + if ece.ece.vendor_id == 0: + return + + rc = v.ibv_set_ece(self.qp, &ece.ece) + if rc != 0: + raise PyverbsRDMAError('Failed to set ECE', rc) + + def query_ece(self): + """ + Query QPs ECE options + :return: ECE object with this QP ece configuration. + """ + ece = ECE() + rc = v.ibv_query_ece(self.qp, &ece.ece) + if rc != 0: + raise PyverbsRDMAError('Failed to query ECE', rc) + return ece + @property def qp_type(self): return self.qp.qp_type