Merge pull request #795 from EdwardSro/pr-memory-window

pyverbs: Extend memory window support

Documentation/pyverbs.md

@@ -175,6 +175,8 @@ with d.Context(name='mlx5_0') as ctx:
 ##### Memory window
 The following example shows the equivalent of creating a type 1 memory window.
 It includes opening a device and allocating the necessary PD.
+The user should unbind or close the memory window before being able to
+deregister an MR that the MW is bound to.
 ```python
 import pyverbs.device as d
 from pyverbs.pd import PD

pyverbs/libibverbs.pxd

@@ -146,7 +146,7 @@ cdef extern from 'infiniband/verbs.h':
         ibv_pd          *pd
         unsigned int    rkey
         unsigned int    handle
-        ibv_mw_type     mw_type
+        ibv_mw_type     type
 
     cdef struct ibv_alloc_dm_attr:
         size_t          length
@@ -303,6 +303,11 @@ cdef extern from 'infiniband/verbs.h':
         unsigned long   length
         unsigned int    mw_access_flags
 
+    cdef struct ibv_mw_bind:
+        uint64_t            wr_id
+        unsigned int        send_flags
+        ibv_mw_bind_info    bind_info
+
     cdef struct bind_mw:
         ibv_mw              *mw
         unsigned int        rkey
@@ -313,10 +318,6 @@ cdef extern from 'infiniband/verbs.h':
         unsigned short  hdr_sz
         unsigned short  mss
 
-    cdef union unnamed:
-        bind_mw         bind_mw
-        tso             tso
-
     cdef struct xrc:
         unsigned int    remote_srqn
 
@@ -329,10 +330,12 @@ cdef extern from 'infiniband/verbs.h':
         ibv_sge         *sg_list
         int             num_sge
         ibv_wr_opcode   opcode
+        uint32_t        imm_data
         unsigned int    send_flags
         wr              wr
         qp_type         qp_type
-        unnamed         unnamed
+        bind_mw         bind_mw
+        tso             tso
 
     cdef struct ibv_qp_cap:
         unsigned int    max_send_wr
@@ -557,6 +560,7 @@ cdef extern from 'infiniband/verbs.h':
     int ibv_destroy_qp(ibv_qp *qp)
     int ibv_post_recv(ibv_qp *qp, ibv_recv_wr *wr, ibv_recv_wr **bad_wr)
     int ibv_post_send(ibv_qp *qp, ibv_send_wr *wr, ibv_send_wr **bad_wr)
+    int ibv_bind_mw(ibv_qp *qp, ibv_mw *mw, ibv_mw_bind *mw_bind)
     ibv_xrcd *ibv_open_xrcd(ibv_context *context,
                             ibv_xrcd_init_attr *xrcd_init_attr)
     int ibv_close_xrcd(ibv_xrcd *xrcd)

pyverbs/mr.pxd

@@ -23,6 +23,10 @@ cdef class MWBindInfo(PyverbsCM):
     cdef v.ibv_mw_bind_info info
     cdef object mr
 
+cdef class MWBind(PyverbsCM):
+    cdef v.ibv_mw_bind mw_bind
+    cdef object mr
+
 cdef class MW(PyverbsCM):
     cdef object pd
     cdef v.ibv_mw *mw

pyverbs/mr.pyx

@@ -222,6 +222,39 @@ cdef class MWBindInfo(PyverbsCM):
         self.info.length = length
         self.info.mw_access_flags = mw_access_flags
 
+    @property
+    def mw_access_flags(self):
+        return self.info.mw_access_flags
+
+    @property
+    def length(self):
+        return self.info.length
+
+    @property
+    def addr(self):
+        return self.info.addr
+
+    def __str__(self):
+        print_format = '{:22}: {:<20}\n'
+        return 'MWBindInfo:\n' +\
+            print_format.format('Addr', self.info.addr) +\
+            print_format.format('Length', self.info.length) +\
+            print_format.format('MW access flags', self.info.mw_access_flags)
+
+
+cdef class MWBind(PyverbsCM):
+    def __init__(self, MWBindInfo info not None,send_flags, wr_id=0):
+        super().__init__()
+        self.mw_bind.wr_id = wr_id
+        self.mw_bind.send_flags = send_flags
+        self.mw_bind.bind_info = info.info
+
+    def __str__(self):
+        print_format = '{:22}: {:<20}\n'
+        return 'MWBind:\n' +\
+            print_format.format('WR id', self.mw_bind.wr_id) +\
+            print_format.format('Send flags', self.mw_bind.send_flags)
+
 
 cdef class MW(PyverbsCM):
     def __init__(self, PD pd not None, v.ibv_mw_type mw_type):
@@ -260,6 +293,25 @@ cdef class MW(PyverbsCM):
             self.mw = NULL
             self.pd = None
 
+    @property
+    def handle(self):
+        return self.mw.handle
+
+    @property
+    def rkey(self):
+        return self.mw.rkey
+
+    @property
+    def type(self):
+        return self.mw.type
+
+    def __str__(self):
+        print_format = '{:22}: {:<20}\n'
+        return 'MW:\n' +\
+            print_format.format('Rkey', self.mw.rkey) +\
+            print_format.format('Handle', self.mw.handle) +\
+            print_format.format('MW Type', mwtype2str(self.mw.type))
+
 
 cdef class DMMR(MR):
     def __init__(self, PD pd not None, length, access, DM dm, offset):

pyverbs/qp.pyx

@@ -8,11 +8,11 @@ import weakref
 from pyverbs.pyverbs_error import PyverbsUserError, PyverbsError, PyverbsRDMAError
 from pyverbs.utils import gid_str, qp_type_to_str, qp_state_to_str, mtu_to_str
 from pyverbs.utils import access_flags_to_str, mig_state_to_str
-from pyverbs.base import PyverbsRDMAErrno
+from pyverbs.mr cimport MW, MWBindInfo, MWBind
 from pyverbs.wr cimport RecvWR, SendWR, SGE
+from pyverbs.base import PyverbsRDMAErrno
 from pyverbs.addr cimport AHAttr, GID, AH
 from pyverbs.base cimport close_weakrefs
-from pyverbs.mr cimport MW, MWBindInfo
 cimport pyverbs.libibverbs_enums as e
 from pyverbs.addr cimport GlobalRoute
 from pyverbs.device cimport Context
@@ -1188,6 +1188,17 @@ cdef class QP(PyverbsCM):
             raise PyverbsRDMAError('Failed to query ECE', rc)
         return ece
 
+    def bind_mw(self, MW mw not None, MWBind mw_bind):
+        """
+        Bind Memory window type 1.
+        :param mw: The memory window to bind.
+        :param mw_bind: MWBind object, includes the bind attributes.
+        :return: None
+        """
+        rc = v.ibv_bind_mw(self.qp, mw.mw, &mw_bind.mw_bind)
+        if rc != 0:
+            raise PyverbsRDMAError('Failed to Bind MW', rc)
+
     @property
     def qp_type(self):
         return self.qp.qp_type

pyverbs/wr.pyx

@@ -2,8 +2,10 @@
 # Copyright (c) 2019 Mellanox Technologies Inc. All rights reserved. See COPYING file
 
 from pyverbs.pyverbs_error import PyverbsUserError, PyverbsError
-from pyverbs.base import PyverbsRDMAErrno
+from pyverbs.base import PyverbsRDMAErrno, inc_rkey
+from pyverbs.mr cimport MW, MR, MWBindInfo
 cimport pyverbs.libibverbs_enums as e
+cimport pyverbs.libibverbs as v
 from pyverbs.addr cimport AH
 from libc.stdlib cimport free, malloc
 from libc.string cimport memcpy
@@ -143,21 +145,25 @@ cdef class RecvWR(PyverbsCM):
 
 
 cdef class SendWR(PyverbsCM):
-    def __init__(self, wr_id=0, opcode=e.IBV_WR_SEND, num_sge=0, sg = None,
-                 send_flags=e.IBV_SEND_SIGNALED, SendWR next_wr = None):
+    def __init__(self, wr_id=0, opcode=e.IBV_WR_SEND, num_sge=0, imm_data=0,
+                 sg = None, send_flags=e.IBV_SEND_SIGNALED,
+                 SendWR next_wr = None):
         """
         Initialize a SendWR object with user-provided or default values.
         :param wr_id: A user-defined WR ID
         :param opcode: The WR's opcode
         :param num_sge: Number of scatter-gather elements in the WR
-        :param send_flags: Send flags as define in ibv_send_flags enum
+        :param imm_data: Immediate data
         :param sg: A SGE element, head of the scatter-gather list
+        :param send_flags: Send flags as define in ibv_send_flags enum
         :return: An initialized SendWR object
         """
         cdef v.ibv_sge *dst
 
         super().__init__()
-        if num_sge < 1 or sg is None:
+        mw_opcodes = [e.IBV_WR_LOCAL_INV, e.IBV_WR_BIND_MW,
+                      e.IBV_WR_SEND_WITH_INV]
+        if opcode not in mw_opcodes and (num_sge < 1 or sg is None):
             raise PyverbsUserError('A WR needs at least one SGE')
         self.send_wr.sg_list = <v.ibv_sge*>malloc(num_sge * sizeof(v.ibv_sge))
         if self.send_wr.sg_list == NULL:
@@ -170,6 +176,7 @@ cdef class SendWR(PyverbsCM):
             self.send_wr.next = &next_wr.send_wr
         self.send_wr.opcode = opcode
         self.send_wr.send_flags = send_flags
+        self.send_wr.imm_data = imm_data
         self.ah = None
 
     def __dealloc(self):
@@ -184,7 +191,8 @@ cdef class SendWR(PyverbsCM):
                print_format.format('Num SGE', self.send_wr.num_sge) +\
                print_format.format('Opcode', self.send_wr.opcode) +\
                print_format.format('Send flags',
-                                   send_flags_to_str(self.send_wr.send_flags))
+                                   send_flags_to_str(self.send_wr.send_flags) +\
+               print_format.format('Imm Data', self.send_wr.imm_data))
 
     @property
     def next_wr(self):
@@ -211,6 +219,13 @@ cdef class SendWR(PyverbsCM):
     def num_sge(self, val):
         self.send_wr.num_sge = val
 
+    @property
+    def imm_data(self):
+        return self.send_wr.imm_data
+    @imm_data.setter
+    def imm_data(self, val):
+        self.send_wr.imm_data = val
+
     @property
     def opcode(self):
         return self.send_wr.opcode
@@ -273,6 +288,23 @@ cdef class SendWR(PyverbsCM):
         self.send_wr.wr.atomic.compare_add = compare_add
         self.send_wr.wr.atomic.swap = swap
 
+    def set_bind_wr(self, MW mw, MWBindInfo bind_info):
+        """
+        Set the members of the bind_mw struct in the send_wr.
+        :param mw: The MW to bind.
+        :param bind_info: MWBindInfo object, includes the bind attributes.
+        :return: None
+        """
+        self.send_wr.bind_mw.mw = mw.mw
+        # Create the new key from the MW rkey.
+        rkey = inc_rkey(mw.rkey)
+        self.send_wr.bind_mw.rkey = rkey
+        self.send_wr.bind_mw.bind_info = bind_info.info
+
+    @property
+    def rkey(self):
+        return self.send_wr.bind_mw.rkey
+
     def set_qp_type_xrc(self, remote_srqn):
         """
         Set the members of the xrc struct in the send_wr's qp_type union, used