pyverbs: Add support for ECE

ECE (enhanced connection establishment) is a mechanism that gives an
option to different libibverbs providers to advertise and use various
provider-specific QP configuration options.
Add those verbs:
ibv_query_ece - get QPs ece.
ibv_set_ece - set the QPs ece.
rdma_set_local_ece - set the local CMs ece.
rdma_get_remote_ece - get the remote CM ece from the connection request.

Signed-off-by: Maxim Chicherin <maximc@mellanox.com>
Signed-off-by: Ido Kalir <idok@mellanox.com>

pyverbs/cmid.pyx

@@ -2,7 +2,7 @@ from libc.stdint cimport uintptr_t
 from libc.string cimport memset
 
 from pyverbs.pyverbs_error import PyverbsUserError
-from pyverbs.qp cimport QPInitAttr, QPAttr
+from pyverbs.qp cimport QPInitAttr, QPAttr, ECE
 from pyverbs.base import PyverbsRDMAErrno
 cimport pyverbs.libibverbs_enums as e
 cimport pyverbs.librdmacm_enums as ce
@@ -536,6 +536,27 @@ cdef class CMID(PyverbsCM):
         if ret != 0:
             raise PyverbsRDMAErrno('Failed to Complete an active connection request')
 
+    def set_local_ece(self, ECE ece):
+        """
+        Set local ECE paraemters to be used for REQ/REP communication.
+        :param ece: ECE object with the requested configuration
+        :return: None
+        """
+        rc = cm.rdma_set_local_ece(self.id, &ece.ece)
+        if rc != 0:
+            raise PyverbsRDMAErrno('Failed to set local ECE')
+
+    def get_remote_ece(self):
+        """
+        Get ECE parameters as were received from the communication peer.
+        :return: ECE object with the ece configuration
+        """
+        ece = ECE()
+        rc = cm.rdma_get_remote_ece(self.id, &ece.ece)
+        if rc != 0:
+            raise PyverbsRDMAErrno('Failed to get remote ECE')
+        return ece
+
     def create_qp(self, QPInitAttr qp_init not None):
         """
         Create a QP, which is associated with CMID.

pyverbs/libibverbs.pxd

@@ -475,6 +475,11 @@ cdef extern from 'infiniband/verbs.h':
         uint64_t        wr_id
         unsigned int    wr_flags
 
+    cdef struct ibv_ece:
+        uint32_t vendor_id
+        uint32_t options
+        uint32_t comp_mask
+
     ibv_device **ibv_get_device_list(int *n)
     void ibv_free_device_list(ibv_device **list)
     ibv_context *ibv_open_device(ibv_device *device)
@@ -601,3 +606,5 @@ cdef extern from 'infiniband/verbs.h':
 cdef extern from 'infiniband/driver.h':
     int ibv_query_gid_type(ibv_context *context, uint8_t port_num,
                            unsigned int index, ibv_gid_type *type)
+    int ibv_set_ece(ibv_qp *qp, ibv_ece *ece)
+    int ibv_query_ece(ibv_qp *qp, ibv_ece *ece)

pyverbs/librdmacm.pxd

@@ -102,6 +102,8 @@ cdef extern from '<rdma/rdma_cma.h>':
     int rdma_create_id(rdma_event_channel *channel, rdma_cm_id **id,
                        void *context, rdma_port_space ps)
     int rdma_destroy_id(rdma_cm_id *id)
+    int rdma_get_remote_ece(rdma_cm_id *id, ibv_ece *ece)
+    int rdma_set_local_ece(rdma_cm_id *id, ibv_ece *ece)
     int rdma_get_request(rdma_cm_id *listen, rdma_cm_id **id)
     int rdma_bind_addr(rdma_cm_id *id, sockaddr *addr)
     int rdma_resolve_addr(rdma_cm_id *id, sockaddr *src_addr,

pyverbs/qp.pxd

@@ -45,3 +45,6 @@ cdef class DataBuffer(PyverbsCM):
 
 cdef class QPEx(QP):
     cdef v.ibv_qp_ex *qp_ex
+
+cdef class ECE(PyverbsCM):
+    cdef v.ibv_ece ece

pyverbs/qp.pyx

@@ -5,9 +5,8 @@ from libc.stdlib cimport malloc, free
 from libc.string cimport memcpy
 import weakref
 
+from pyverbs.pyverbs_error import PyverbsUserError, PyverbsError, PyverbsRDMAError
 from pyverbs.utils import gid_str, qp_type_to_str, qp_state_to_str, mtu_to_str
-from pyverbs.pyverbs_error import PyverbsUserError, PyverbsError, \
-    PyverbsRDMAError
 from pyverbs.utils import access_flags_to_str, mig_state_to_str
 from pyverbs.base import PyverbsRDMAErrno
 from pyverbs.wr cimport RecvWR, SendWR, SGE
@@ -873,6 +872,28 @@ cdef class QPAttr(PyverbsObject):
                print_format.format('Rate limit', self.attr.rate_limit)
 
 
+cdef class ECE(PyverbsCM):
+    def __init__(self, vendor_id=0, options=0, comp_mask=0):
+        """
+        :param vendor_id: Unique identifier of the provider vendor.
+        :param options: Provider specific attributes which are supported or
+                        needed to be enabled by ECE users.
+        :param comp_mask: A bitmask specifying which ECE options should be
+                          valid.
+        """
+        super().__init__()
+        self.ece.vendor_id = vendor_id
+        self.ece.options = options
+        self.ece.comp_mask = comp_mask
+
+    def __str__(self):
+        print_format = '{:22}: 0x{:<20x}\n'
+        return 'ECE:\n' +\
+        print_format.format('Vendor ID', self.ece.vendor_id) +\
+        print_format.format('Options', self.ece.options) +\
+        print_format.format('Comp Mask', self.ece.comp_mask)
+
+
 cdef class QP(PyverbsCM):
     def __init__(self, object creator not None, object init_attr not None,
                  QPAttr qp_attr=None):
@@ -1143,6 +1164,30 @@ cdef class QP(PyverbsCM):
                 memcpy(&bad_wr.send_wr, my_bad_wr, sizeof(bad_wr.send_wr))
             raise PyverbsRDMAError('Failed to post send', rc)
 
+    def set_ece(self, ECE ece):
+        """
+        Set ECE options and use them for QP configuration stage
+        :param ece: The requested ECE values.
+        :return: None
+        """
+        if ece.ece.vendor_id == 0:
+            return
+
+        rc = v.ibv_set_ece(self.qp, &ece.ece)
+        if rc != 0:
+            raise PyverbsRDMAError('Failed to set ECE', rc)
+
+    def query_ece(self):
+        """
+        Query QPs ECE options
+        :return: ECE object with this QP ece configuration.
+        """
+        ece = ECE()
+        rc = v.ibv_query_ece(self.qp, &ece.ece)
+        if rc != 0:
+            raise PyverbsRDMAError('Failed to query ECE', rc)
+        return ece
+
     @property
     def qp_type(self):
         return self.qp.qp_type