pyverbs: Introducing MR object

MR represnts ibv_mr.

Instead of allocating a buffer prior to calling ibv_reg_mr, the user
provides the MR class the required size and flags, and pyverbs
allocates the needed buffer.

The MR class exposes read() and write() methods to users to allow
setting and reading the underlying memory buffer.

Signed-off-by: Noa Osherovich <noaos@mellanox.com>

pyverbs/CMakeLists.txt

@@ -6,6 +6,7 @@ rdma_cython_module(pyverbs
   base.pyx
   device.pyx
   enums.pyx
+  mr.pyx
   pd.pyx
   )
 

pyverbs/libibverbs.pxd

@@ -68,6 +68,15 @@ cdef extern from 'infiniband/verbs.h':
         ibv_context     *context
         unsigned int    handle
 
+    cdef struct ibv_mr:
+        ibv_context     *context
+        ibv_pd          *pd
+        void            *addr
+        size_t          length
+        unsigned int    handle
+        unsigned int    lkey
+        unsigned int    rkey
+
     ibv_device **ibv_get_device_list(int *n)
     void ibv_free_device_list(ibv_device **list)
     ibv_context *ibv_open_device(ibv_device *device)
@@ -78,3 +87,5 @@ cdef extern from 'infiniband/verbs.h':
                       int index, ibv_gid *gid)
     ibv_pd *ibv_alloc_pd(ibv_context *context)
     int ibv_dealloc_pd(ibv_pd *pd)
+    ibv_mr *ibv_reg_mr(ibv_pd *pd, void *addr, size_t length, int access)
+    int ibv_dereg_mr(ibv_mr *mr)

pyverbs/mr.pxd

@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file
+
+from pyverbs.base cimport PyverbsCM
+from . cimport libibverbs as v
+
+
+cdef class MR(PyverbsCM):
+    cdef object pd
+    cdef v.ibv_mr *mr
+    cdef void *buf
+    cpdef read(self, length, offset)

pyverbs/mr.pyx

@@ -0,0 +1,109 @@
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file
+
+from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError
+from pyverbs.base import PyverbsRDMAErrno
+from .pd cimport PD
+import resource
+
+cdef extern from 'stdlib.h':
+    int posix_memalign(void **memptr, size_t alignment, size_t size)
+cdef extern from 'stdlib.h':
+    void free(void *ptr)
+cdef extern from 'string.h':
+    void *memcpy(void *dest, const void *src, size_t n)
+    void *memset(void *s, int c, size_t n)
+cdef extern from 'stdint.h':
+    ctypedef int uintptr_t
+
+
+cdef class MR(PyverbsCM):
+    """
+    MR class represents ibv_mr. Buffer allocation in done in the c'tor. Freeing
+    it is done in close().
+    """
+    def __cinit__(self, PD pd not None, length, access):
+        """
+        Allocate a user-level buffer of length <length> and register a Memory
+        Region of the given length and access flags.
+        :param pd: A PD object
+        :param length: Length in bytes
+        :param access: Access flags, see ibv_access_flags enum
+        :return: The newly created MR on success
+        """
+        #We want to enable registering an MR of size 0 but this fails with a
+        #buffer of size 0, so in this case lets increase the buffer
+        if length == 0:
+            length = 10
+        rc = posix_memalign(&self.buf, resource.getpagesize(), length)
+        if rc:
+            raise PyverbsRDMAError('Failed to allocate MR buffer of size {l}'.
+                                   format(l=length))
+        memset(self.buf, 0, length)
+        self.mr = v.ibv_reg_mr(<v.ibv_pd*>pd.pd, self.buf, length, access)
+        if self.mr == NULL:
+            raise PyverbsRDMAErrno('Failed to register a MR. length: {l}, access flags: {a}'.
+                                   format(l=length, a=access))
+        self.pd = pd
+        pd.add_ref(self)
+        self.logger.debug('Registered ibv_mr. Length: {l}, access flags {a}'.
+                          format(l=length, a=access))
+
+    def __dealloc__(self):
+        self.close()
+
+    cpdef close(self):
+        """
+        Closes the underlying C object of the MR and frees the memory allocated.
+        MR may be deleted directly or indirectly by closing its context, which
+        leaves the Python PD object without the underlying C object, so during
+        destruction, need to check whether or not the C object exists.
+        :return: None
+        """
+        self.logger.debug('Closing MR')
+        if self.mr != NULL:
+            rc = v.ibv_dereg_mr(self.mr)
+            if rc != 0:
+                raise PyverbsRDMAErrno('Failed to dereg MR')
+            self.mr = NULL
+            self.pd = None
+        free(self.buf)
+        self.buf = NULL
+
+    def write(self, data, length):
+        """
+        Write user data to the MR's buffer using memcpy
+        :param data: User data to write
+        :param length: Length of the data to write
+        :return: None
+        """
+        # If data is a string, cast it to bytes as Python3 doesn't
+        # automatically convert it.
+        if isinstance(data, str):
+            data = data.encode()
+        memcpy(self.buf, <char *>data, length)
+
+    cpdef read(self, length, offset):
+        """
+        Reads data from the MR's buffer
+        :param length: Length of data to read
+        :param offset: Reading offset
+        :return: The data on the buffer in the requested offset
+        """
+        cdef char *data
+        cdef int off = offset # we can't use offset in the next line, as it is
+                              # a Python object and not C
+        data = <char*>(self.buf + off)
+        return data[:length]
+
+    @property
+    def buf(self):
+        return <uintptr_t>self.buf
+
+    @property
+    def lkey(self):
+        return self.mr.lkey
+
+    @property
+    def rkey(self):
+        return self.mr.rkey

pyverbs/pd.pxd

@@ -8,3 +8,5 @@ from .base cimport PyverbsCM
 cdef class PD(PyverbsCM):
     cdef v.ibv_pd *pd
     cdef Context ctx
+    cdef add_ref(self, obj)
+    cdef object mrs

pyverbs/pd.pyx

@@ -1,7 +1,10 @@
 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
 # Copyright (c) 2019, Mellanox Technologies. All rights reserved.
-from pyverbs.pyverbs_error import PyverbsRDMAError
+import weakref
+
+from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError
 from pyverbs.base import PyverbsRDMAErrno
+from .mr cimport MR
 
 cdef extern from 'errno.h':
     int errno
@@ -21,6 +24,7 @@ cdef class PD(PyverbsCM):
         self.ctx = context
         context.add_ref(self)
         self.logger.debug('PD: Allocated ibv_pd')
+        self.mrs = weakref.WeakSet()
 
     def __dealloc__(self):
         """
@@ -38,9 +42,16 @@ cdef class PD(PyverbsCM):
         :return: None
         """
         self.logger.debug('Closing PD')
+        self.close_weakrefs([self.mrs])
         if self.pd != NULL:
             rc = v.ibv_dealloc_pd(self.pd)
             if rc != 0:
                 raise PyverbsRDMAErrno('Failed to dealloc PD')
             self.pd = NULL
             self.ctx = None
+
+    cdef add_ref(self, obj):
+        if isinstance(obj, MR):
+            self.mrs.add(obj)
+        else:
+            raise PyverbsError('Unrecognized object type')