pyverbs: Add memory window support

Add support to create and use Memory Window.
Supports both types of Memory Window (MW type 1 and 2).

Signed-off-by: Ido Kalir <idok@mellanox.com>
Signed-off-by: Edward Srouji <edwards@mellanox.com>

pyverbs/libibverbs.pxd

@@ -146,7 +146,7 @@ cdef extern from 'infiniband/verbs.h':
         ibv_pd          *pd
         unsigned int    rkey
         unsigned int    handle
-        ibv_mw_type     mw_type
+        ibv_mw_type     type
 
     cdef struct ibv_alloc_dm_attr:
         size_t          length
@@ -303,6 +303,11 @@ cdef extern from 'infiniband/verbs.h':
         unsigned long   length
         unsigned int    mw_access_flags
 
+    cdef struct ibv_mw_bind:
+        uint64_t            wr_id
+        unsigned int        send_flags
+        ibv_mw_bind_info    bind_info
+
     cdef struct bind_mw:
         ibv_mw              *mw
         unsigned int        rkey
@@ -313,10 +318,6 @@ cdef extern from 'infiniband/verbs.h':
         unsigned short  hdr_sz
         unsigned short  mss
 
-    cdef union unnamed:
-        bind_mw         bind_mw
-        tso             tso
-
     cdef struct xrc:
         unsigned int    remote_srqn
 
@@ -333,7 +334,8 @@ cdef extern from 'infiniband/verbs.h':
         unsigned int    send_flags
         wr              wr
         qp_type         qp_type
-        unnamed         unnamed
+        bind_mw         bind_mw
+        tso             tso
 
     cdef struct ibv_qp_cap:
         unsigned int    max_send_wr
@@ -557,6 +559,7 @@ cdef extern from 'infiniband/verbs.h':
     int ibv_destroy_qp(ibv_qp *qp)
     int ibv_post_recv(ibv_qp *qp, ibv_recv_wr *wr, ibv_recv_wr **bad_wr)
     int ibv_post_send(ibv_qp *qp, ibv_send_wr *wr, ibv_send_wr **bad_wr)
+    int ibv_bind_mw(ibv_qp *qp, ibv_mw *mw, ibv_mw_bind *mw_bind)
     ibv_xrcd *ibv_open_xrcd(ibv_context *context,
                             ibv_xrcd_init_attr *xrcd_init_attr)
     int ibv_close_xrcd(ibv_xrcd *xrcd)

pyverbs/mr.pxd

@@ -21,6 +21,10 @@ cdef class MWBindInfo(PyverbsCM):
     cdef v.ibv_mw_bind_info info
     cdef object mr
 
+cdef class MWBind(PyverbsCM):
+    cdef v.ibv_mw_bind mw_bind
+    cdef object mr
+
 cdef class MW(PyverbsCM):
     cdef object pd
     cdef v.ibv_mw *mw

pyverbs/mr.pyx

@@ -204,6 +204,39 @@ cdef class MWBindInfo(PyverbsCM):
         self.info.length = length
         self.info.mw_access_flags = mw_access_flags
 
+    @property
+    def mw_access_flags(self):
+        return self.info.mw_access_flags
+
+    @property
+    def length(self):
+        return self.info.length
+
+    @property
+    def addr(self):
+        return self.info.addr
+
+    def __str__(self):
+        print_format = '{:22}: {:<20}\n'
+        return 'MWBindInfo:\n' +\
+            print_format.format('Addr', self.info.addr) +\
+            print_format.format('Length', self.info.length) +\
+            print_format.format('MW access flags', self.info.mw_access_flags)
+
+
+cdef class MWBind(PyverbsCM):
+    def __init__(self, MWBindInfo info not None,send_flags, wr_id=0):
+        super().__init__()
+        self.mw_bind.wr_id = wr_id
+        self.mw_bind.send_flags = send_flags
+        self.mw_bind.bind_info = info.info
+
+    def __str__(self):
+        print_format = '{:22}: {:<20}\n'
+        return 'MWBind:\n' +\
+            print_format.format('WR id', self.mw_bind.wr_id) +\
+            print_format.format('Send flags', self.mw_bind.send_flags)
+
 
 cdef class MW(PyverbsCM):
     def __init__(self, PD pd not None, v.ibv_mw_type mw_type):
@@ -242,6 +275,25 @@ cdef class MW(PyverbsCM):
             self.mw = NULL
             self.pd = None
 
+    @property
+    def handle(self):
+        return self.mw.handle
+
+    @property
+    def rkey(self):
+        return self.mw.rkey
+
+    @property
+    def type(self):
+        return self.mw.type
+
+    def __str__(self):
+        print_format = '{:22}: {:<20}\n'
+        return 'MW:\n' +\
+            print_format.format('Rkey', self.mw.rkey) +\
+            print_format.format('Handle', self.mw.handle) +\
+            print_format.format('MW Type', mwtype2str(self.mw.type))
+
 
 cdef class DMMR(MR):
     def __init__(self, PD pd not None, length, access, DM dm, offset):

pyverbs/qp.pyx

@@ -8,11 +8,11 @@ import weakref
 from pyverbs.pyverbs_error import PyverbsUserError, PyverbsError, PyverbsRDMAError
 from pyverbs.utils import gid_str, qp_type_to_str, qp_state_to_str, mtu_to_str
 from pyverbs.utils import access_flags_to_str, mig_state_to_str
-from pyverbs.base import PyverbsRDMAErrno
+from pyverbs.mr cimport MW, MWBindInfo, MWBind
 from pyverbs.wr cimport RecvWR, SendWR, SGE
+from pyverbs.base import PyverbsRDMAErrno
 from pyverbs.addr cimport AHAttr, GID, AH
 from pyverbs.base cimport close_weakrefs
-from pyverbs.mr cimport MW, MWBindInfo
 cimport pyverbs.libibverbs_enums as e
 from pyverbs.addr cimport GlobalRoute
 from pyverbs.device cimport Context
@@ -1188,6 +1188,17 @@ cdef class QP(PyverbsCM):
             raise PyverbsRDMAError('Failed to query ECE', rc)
         return ece
 
+    def bind_mw(self, MW mw not None, MWBind mw_bind):
+        """
+        Bind Memory window type 1.
+        :param mw: The memory window to bind.
+        :param mw_bind: MWBind object, includes the bind attributes.
+        :return: None
+        """
+        rc = v.ibv_bind_mw(self.qp, mw.mw, &mw_bind.mw_bind)
+        if rc != 0:
+            raise PyverbsRDMAError('Failed to Bind MW', rc)
+
     @property
     def qp_type(self):
         return self.qp.qp_type

pyverbs/wr.pyx

@@ -2,8 +2,10 @@
 # Copyright (c) 2019 Mellanox Technologies Inc. All rights reserved. See COPYING file
 
 from pyverbs.pyverbs_error import PyverbsUserError, PyverbsError
-from pyverbs.base import PyverbsRDMAErrno
+from pyverbs.base import PyverbsRDMAErrno, inc_rkey
+from pyverbs.mr cimport MW, MR, MWBindInfo
 cimport pyverbs.libibverbs_enums as e
+cimport pyverbs.libibverbs as v
 from pyverbs.addr cimport AH
 from libc.stdlib cimport free, malloc
 from libc.string cimport memcpy
@@ -159,7 +161,9 @@ cdef class SendWR(PyverbsCM):
         cdef v.ibv_sge *dst
 
         super().__init__()
-        if num_sge < 1 or sg is None:
+        mw_opcodes = [e.IBV_WR_LOCAL_INV, e.IBV_WR_BIND_MW,
+                      e.IBV_WR_SEND_WITH_INV]
+        if opcode not in mw_opcodes and (num_sge < 1 or sg is None):
             raise PyverbsUserError('A WR needs at least one SGE')
         self.send_wr.sg_list = <v.ibv_sge*>malloc(num_sge * sizeof(v.ibv_sge))
         if self.send_wr.sg_list == NULL:
@@ -284,6 +288,23 @@ cdef class SendWR(PyverbsCM):
         self.send_wr.wr.atomic.compare_add = compare_add
         self.send_wr.wr.atomic.swap = swap
 
+    def set_bind_wr(self, MW mw, MWBindInfo bind_info):
+        """
+        Set the members of the bind_mw struct in the send_wr.
+        :param mw: The MW to bind.
+        :param bind_info: MWBindInfo object, includes the bind attributes.
+        :return: None
+        """
+        self.send_wr.bind_mw.mw = mw.mw
+        # Create the new key from the MW rkey.
+        rkey = inc_rkey(mw.rkey)
+        self.send_wr.bind_mw.rkey = rkey
+        self.send_wr.bind_mw.bind_info = bind_info.info
+
+    @property
+    def rkey(self):
+        return self.send_wr.bind_mw.rkey
+
     def set_qp_type_xrc(self, remote_srqn):
         """
         Set the members of the xrc struct in the send_wr's qp_type union, used