pyverbs: Add Tag matching support

Add OpsWr class to define ibv_ops_wr.
Add enums to support SRQ-TM.
Add flags and opcode enums to support tag-matching operations.
Expose ibv_post_srq_ops() in pyverbs which enables post recv operation
in SRQ-TM.

Signed-off-by: Hadas Abraham <habraham@nvidia.com>
Signed-off-by: Edward Srouji <edwards@nvidia.com>

pyverbs/libibverbs.pxd

@@ -115,6 +115,10 @@ cdef extern from 'infiniband/verbs.h':
         unsigned int    max_ops
         unsigned int    max_sge
 
+    cdef struct ibv_tm_cap:
+        uint32_t        max_num_tags
+        uint32_t        max_ops
+
     cdef struct ibv_cq_moderation_caps:
         unsigned int    max_cq_count
         unsigned int    max_cq_period
@@ -280,6 +284,25 @@ cdef extern from 'infiniband/verbs.h':
         ibv_sge         *sg_list
         int             num_sge
 
+    cdef struct _add:
+        uint64_t        recv_wr_id
+        ibv_sge         *sg_list
+        int             num_sge
+        uint64_t        tag
+        uint64_t        mask
+
+    cdef struct _tm:
+        uint32_t        unexpected_cnt
+        uint32_t        handle
+        _add             add
+
+    cdef struct ibv_ops_wr:
+        uint64_t            wr_id
+        ibv_ops_wr          *next
+        ibv_ops_wr_opcode   opcode
+        int                 flags
+        _tm                  tm
+
     cdef struct rdma:
         unsigned long   remote_addr
         unsigned int    rkey
@@ -381,7 +404,7 @@ cdef extern from 'infiniband/verbs.h':
         ibv_pd          *pd
         ibv_xrcd        *xrcd
         ibv_cq          *cq
-        ibv_tm_caps      tm_cap
+        ibv_tm_cap      tm_cap
 
     cdef struct ibv_srq:
         ibv_context     *context
@@ -711,6 +734,7 @@ cdef extern from 'infiniband/verbs.h':
     int ibv_destroy_srq(ibv_srq *srq)
     int ibv_post_srq_recv(ibv_srq *srq, ibv_recv_wr *recv_wr,
                           ibv_recv_wr **bad_recv_wr)
+    int ibv_post_srq_ops(ibv_srq *srq, ibv_ops_wr *op, ibv_ops_wr **bad_op)
     ibv_pd *ibv_alloc_parent_domain(ibv_context *context,
                                     ibv_parent_domain_init_attr *attr)
     uint32_t ibv_inc_rkey(uint32_t rkey)

pyverbs/libibverbs_enums.pxd

@@ -141,13 +141,25 @@ cdef extern from '<infiniband/verbs.h>':
         IBV_WR_SEND_WITH_INV
         IBV_WR_TSO
 
+    cpdef enum ibv_ops_wr_opcode:
+        IBV_WR_TAG_ADD
+        IBV_WR_TAG_DEL
+        IBV_WR_TAG_SYNC
+
+    cpdef enum ibv_ops_flags:
+        IBV_OPS_SIGNALED
+        IBV_OPS_TM_SYNC
+
     cpdef enum ibv_send_flags:
         IBV_SEND_FENCE
         IBV_SEND_SIGNALED
         IBV_SEND_SOLICITED
         IBV_SEND_INLINE
         IBV_SEND_IP_CSUM
 
+    cpdef enum ibv_tm_cap_flags:
+        IBV_TM_CAP_RC
+
     cpdef enum ibv_qp_type:
         IBV_QPT_RC
         IBV_QPT_UC
@@ -194,6 +206,8 @@ cdef extern from '<infiniband/verbs.h>':
         IBV_WC_FATAL_ERR
         IBV_WC_RESP_TIMEOUT_ERR
         IBV_WC_GENERAL_ERR
+        IBV_WC_TM_ERR
+        IBV_WC_TM_RNDV_INCOMPLETE
 
     cpdef enum ibv_wc_opcode:
         IBV_WC_SEND
@@ -206,6 +220,11 @@ cdef extern from '<infiniband/verbs.h>':
         IBV_WC_TSO
         IBV_WC_RECV
         IBV_WC_RECV_RDMA_WITH_IMM
+        IBV_WC_TM_ADD
+        IBV_WC_TM_DEL
+        IBV_WC_TM_SYNC
+        IBV_WC_TM_RECV
+        IBV_WC_TM_NO_TAG
         IBV_WC_DRIVER2
         IBV_WC_DRIVER3
 
@@ -220,13 +239,17 @@ cdef extern from '<infiniband/verbs.h>':
         IBV_WC_EX_WITH_COMPLETION_TIMESTAMP
         IBV_WC_EX_WITH_CVLAN
         IBV_WC_EX_WITH_FLOW_TAG
+        IBV_WC_EX_WITH_TM_INFO
         IBV_WC_EX_WITH_COMPLETION_TIMESTAMP_WALLCLOCK
 
     cpdef enum ibv_wc_flags:
         IBV_WC_GRH
         IBV_WC_WITH_IMM
         IBV_WC_IP_CSUM_OK
         IBV_WC_WITH_INV
+        IBV_WC_TM_SYNC_REQ
+        IBV_WC_TM_MATCH
+        IBV_WC_TM_DATA_VALID
 
     cpdef enum ibv_srq_attr_mask:
         IBV_SRQ_MAX_WR
@@ -235,12 +258,14 @@ cdef extern from '<infiniband/verbs.h>':
     cpdef enum ibv_srq_type:
         IBV_SRQT_BASIC
         IBV_SRQT_XRC
+        IBV_SRQT_TM
 
     cpdef enum ibv_srq_init_attr_mask:
         IBV_SRQ_INIT_ATTR_TYPE
         IBV_SRQ_INIT_ATTR_PD
         IBV_SRQ_INIT_ATTR_XRCD
         IBV_SRQ_INIT_ATTR_CQ
+        IBV_SRQ_INIT_ATTR_TM
 
     cpdef enum ibv_mig_state:
         IBV_MIG_MIGRATED
@@ -476,3 +501,10 @@ cdef extern from '<infiniband/driver.h>':
     cpdef enum ibv_gid_type_sysfs:
         IBV_GID_TYPE_SYSFS_IB_ROCE_V1
         IBV_GID_TYPE_SYSFS_ROCE_V2
+
+cdef extern from "<infiniband/tm_types.h>":
+    cpdef enum ibv_tmh_op:
+        IBV_TMH_NO_TAG
+        IBV_TMH_RNDV
+        IBV_TMH_FIN
+        IBV_TMH_EAGER

pyverbs/srq.pxd

@@ -18,6 +18,9 @@ cdef class SrqInitAttrEx(PyverbsObject):
     cdef object _pd
     cdef object _xrcd
 
+cdef class OpsWr(PyverbsCM):
+    cdef v.ibv_ops_wr ops_wr
+
 cdef class SRQ(PyverbsCM):
     cdef v.ibv_srq *srq
     cdef object cq

pyverbs/srq.pyx

@@ -1,16 +1,18 @@
 import weakref
+from libc.errno cimport errno
+from libc.string cimport memcpy
+from libc.stdlib cimport malloc, free
 from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError
+from pyverbs.wr cimport RecvWR, SGE, copy_sg_array
 from pyverbs.base import PyverbsRDMAErrno
 from pyverbs.base cimport close_weakrefs
+cimport pyverbs.libibverbs_enums as e
 from pyverbs.device cimport Context
 from pyverbs.cq cimport CQEX, CQ
+cimport pyverbs.libibverbs as v
 from pyverbs.xrcd cimport XRCD
-from pyverbs.wr cimport RecvWR
 from pyverbs.qp cimport QP
 from pyverbs.pd cimport PD
-from libc.errno cimport errno
-from libc.string cimport memcpy
-
 
 cdef class SrqAttr(PyverbsObject):
     def __init__(self, max_wr=100, max_sge=1, srq_limit=0):
@@ -114,6 +116,20 @@ cdef class SrqInitAttrEx(PyverbsObject):
         self._xrcd = val
         self.attr.xrcd = val.xrcd
 
+    @property
+    def max_num_tags(self):
+        return self.attr.tm_cap.max_num_tags
+    @max_num_tags.setter
+    def max_num_tags(self, val):
+        self.attr.tm_cap.max_num_tags = val
+
+    @property
+    def max_ops(self):
+        return self.attr.tm_cap.max_ops
+    @max_ops.setter
+    def max_ops(self, val):
+        self.attr.tm_cap.max_ops = val
+
     @property
     def cq(self):
         return self._cq
@@ -127,6 +143,105 @@ cdef class SrqInitAttrEx(PyverbsObject):
             self._cq = val
 
 
+cdef class OpsWr(PyverbsCM):
+    def __init__(self, wr_id=0, opcode=e.IBV_WR_TAG_ADD, flags=e.IBV_OPS_SIGNALED,
+                 OpsWr next_wr=None, unexpected_cnt=0, recv_wr_id=0,
+                 num_sge=None, tag=0, mask=0, sg_list=None):
+        self.ops_wr.wr_id = wr_id
+        self.ops_wr.opcode = opcode
+        self.ops_wr.flags = flags
+        self.ops_wr.tm.unexpected_cnt = unexpected_cnt
+        self.ops_wr.tm.add.recv_wr_id = recv_wr_id
+        self.ops_wr.tm.add.tag = tag
+        self.ops_wr.tm.add.mask = mask
+        if next_wr is not None:
+            self.ops_wr.next = &next_wr.ops_wr
+        if num_sge is not None:
+            self.ops_wr.tm.add.num_sge = num_sge
+        cdef v.ibv_sge *dst
+        if sg_list is not None:
+            self.ops_wr.tm.add.sg_list = <v.ibv_sge*>malloc(num_sge * sizeof(v.ibv_sge))
+            if self.ops_wr.tm.add.sg_list == NULL:
+                raise MemoryError('Failed to malloc SG buffer')
+            dst = self.ops_wr.tm.add.sg_list
+            copy_sg_array(dst, sg_list, num_sge)
+
+    def __dealloc__(self):
+        self.close()
+
+    cpdef close(self):
+        if self.ops_wr.tm.add.sg_list != NULL:
+            free(self.ops_wr.tm.add.sg_list)
+            self.ops_wr.tm.add.sg_list = NULL
+
+    @property
+    def wr_id(self):
+        return self.ops_wr.wr_id
+    @wr_id.setter
+    def wr_id(self, val):
+        self.ops_wr.wr_id = val
+
+    @property
+    def next_wr(self):
+        if self.ops_wr.next == NULL:
+            return None
+        val = OpsWr()
+        val.ops_wr = self.ops_wr.next[0]
+        return val
+    @next_wr.setter
+    def next_wr(self, OpsWr val not None):
+        self.ops_wr.next = &val.ops_wr
+
+    @property
+    def opcode(self):
+        return self.ops_wr.opcode
+    @opcode.setter
+    def opcode(self, val):
+        self.ops_wr.opcode = val
+
+    @property
+    def flags(self):
+        return self.ops_wr.flags
+    @flags.setter
+    def flags(self, val):
+        self.ops_wr.flags = val
+
+    @property
+    def unexpected_cnt(self):
+        return self.ops_wr.tm.unexpected_cnt
+    @unexpected_cnt.setter
+    def unexpected_cnt(self, val):
+        self.ops_wr.tm.unexpected_cnt = val
+
+    @property
+    def recv_wr_id(self):
+        return self.ops_wr.tm.add.recv_wr_id
+    @recv_wr_id.setter
+    def recv_wr_id(self, val):
+        self.ops_wr.tm.add.recv_wr_id = val
+
+    @property
+    def tag(self):
+        return self.ops_wr.tm.add.tag
+    @tag.setter
+    def tag(self, val):
+        self.ops_wr.tm.add.tag = val
+
+    @property
+    def handle(self):
+        return self.ops_wr.tm.handle
+    @handle.setter
+    def handle(self, val):
+        self.ops_wr.tm.handle = val
+
+    @property
+    def mask(self):
+        return self.ops_wr.tm.add.mask
+    @mask.setter
+    def mask(self, val):
+        self.ops_wr.tm.add.mask = val
+
+
 cdef class SRQ(PyverbsCM):
     def __init__(self, object creator not None, object attr not None):
         super().__init__()
@@ -200,6 +315,20 @@ cdef class SRQ(PyverbsCM):
             raise PyverbsRDMAError('Failed to query SRQ', rc)
         return attr
 
+    def post_srq_ops(self, OpsWr wr not None, OpsWr bad_wr=None):
+        """
+        Perform on a special shared receive queue (SRQ) configuration manipulations
+        :param wr: Ops Work Requests to be posted to the TM-Shared Receive Queue
+        :param bad_wr: A pointer that will be filled with the first Ops Work Request,
+                       that its processing failed
+        """
+        cdef v.ibv_ops_wr *my_bad_wr
+        rc= v.ibv_post_srq_ops(self.srq, &wr.ops_wr, &my_bad_wr)
+        if rc != 0:
+            if bad_wr:
+                memcpy(&bad_wr.ops_wr, my_bad_wr, sizeof(bad_wr.ops_wr))
+            raise PyverbsRDMAError('post SRQ ops failed.', rc)
+
     def post_recv(self, RecvWR wr not None, RecvWR bad_wr=None):
         cdef v.ibv_recv_wr *my_bad_wr
         rc = v.ibv_post_srq_recv(self.srq, &wr.recv_wr, &my_bad_wr)