pyverbs: Introduce PD class

This patch introduces PD object which represents ibv_pd.

This commit also introduces the usage of weak references.
In Python, a weak reference is a reference to an object not strong
enough to keep it from being garbage-collected.
In Pyverbs, it is used to allow an object to keep track of the objects
created using it, so if an object is being destroyed (e.g. explicitly
by the user), it can iterate over the dependent objects and destroy
the underlying C objects.
E.g.: A user opens a device and creates a PD. If the Context object
is being destroyed before the PD object, the kernel will return
EBUSY. To avoid that, the Context object will hold a weakref to the
PD and destroy the C PD before closing itself.
This renders the Python PD object useless, but it is a user's choice
that we are not blocking.

In order to provide a clean teardown, Pyverbs Context is also a
context manager, which means itshouldbe used within a 'with' block.
This way, when the block is over, teardown will be executed properly.
The same applies for all other relevant pyverbs' classes.
A PyverbsCM class was added as an alternative base objects to inherit
from. It inherits from PyverbsObject and adds the __enter__ and
__exit__ functions needed for a Python context manager.

Signed-off-by: Noa Osherovich <noaos@mellanox.com>

pyverbs/CMakeLists.txt

@@ -1,11 +1,12 @@
 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
-# Copyright (c) 2018, Mellanox Technologies. All rights reserved. See COPYING file
+# Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file
 
 rdma_cython_module(pyverbs
-  enums.pyx
+  addr.pyx
   base.pyx
   device.pyx
-  addr.pyx
+  enums.pyx
+  pd.pyx
   )
 
 rdma_python_module(pyverbs

pyverbs/base.pxd

@@ -1,5 +1,9 @@
 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
-# Copyright (c) 2018, Mellanox Technologies. All rights reserved.
+# Copyright (c) 2019, Mellanox Technologies. All rights reserved.
 
 cdef class PyverbsObject(object):
+    cdef object __weakref__
     cdef object logger
+
+cdef class PyverbsCM(PyverbsObject):
+    cpdef close(self)

pyverbs/base.pyx

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
-# Copyright (c) 2018, Mellanox Technologies. All rights reserved.
+# Copyright (c) 2019, Mellanox Technologies. All rights reserved.
 
 import logging
 from pyverbs.pyverbs_error import PyverbsRDMAError
@@ -22,3 +22,42 @@ cdef class PyverbsObject(object):
 
     def set_log_level(self, val):
         self.logger.setLevel(val)
+
+    def close_weakrefs(self, iterables):
+        """
+        For each iterable element of iterables, pop each element and
+        call its close() method. This method is used when an object is being
+        closed while other objects still hold C references to it; the object
+        holds weakrefs to such other object, and closes them before trying to
+        teardown the C resources.
+        :param iterables: an array of WeakSets
+        :return: None
+        """
+        # None elements can be present if an object's close() was called more
+        # than once (e.g. GC and by another object)
+        for it in iterables:
+            if it is None:
+                continue
+            while True:
+                try:
+                    tmp = it.pop()
+                    tmp.close()
+                except KeyError: # popping an empty set
+                    break
+
+
+cdef class PyverbsCM(PyverbsObject):
+    """
+    This is a base class for pyverbs' context manager objects. It includes
+    __enter__ and __exit__ functions.
+    close() is also declared but it should be overridden by each inheriting
+    class.
+    """
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        return self.close()
+
+    cpdef close(self):
+        pass

pyverbs/device.pxd

@@ -1,14 +1,15 @@
 # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
 # Copyright (c) 2018, Mellanox Technologies. All rights reserved. See COPYING file
 
-from .base cimport PyverbsObject
+from .base cimport PyverbsObject, PyverbsCM
 cimport pyverbs.libibverbs as v
 
 
-cdef class Context(PyverbsObject):
+cdef class Context(PyverbsCM):
     cdef v.ibv_context *context
     cdef object name
-    cpdef close(self)
+    cdef add_ref(self, obj)
+    cdef object pds
 
 cdef class DeviceAttr(PyverbsObject):
     cdef v.ibv_device_attr dev_attr

pyverbs/device.pyx

@@ -6,11 +6,14 @@ Device module introduces the Context and DeviceAttr class.
 It allows user to open an IB device (using Context(name=<name>) and query it,
 which returns a DeviceAttr object.
 """
-from .pyverbs_error import PyverbsRDMAError
+import weakref
+
+from .pyverbs_error import PyverbsRDMAError, PyverbsError
 from .pyverbs_error import PyverbsUserError
 from pyverbs.base import PyverbsRDMAErrno
 cimport pyverbs.libibverbs as v
 from pyverbs.addr cimport GID
+from pyverbs.pd cimport PD
 
 cdef extern from 'errno.h':
     int errno
@@ -55,10 +58,9 @@ class Device(PyverbsObject):
                 guid=guid_to_hex(self.guid))
 
 
-cdef class Context(PyverbsObject):
+cdef class Context(PyverbsCM):
     """
-    Context class represents the C ibv_context. It currently allows only
-    querying the underlying device.
+    Context class represents the C ibv_context.
     """
     def __cinit__(self, **kwargs):
         """
@@ -71,6 +73,8 @@ cdef class Context(PyverbsObject):
         """
         cdef int count
         cdef v.ibv_device **dev_list
+
+        self.pds = weakref.WeakSet()
         dev_name = kwargs.get('name')
 
         if dev_name is not None:
@@ -106,6 +110,7 @@ cdef class Context(PyverbsObject):
 
     cpdef close(self):
         self.logger.debug('Closing Context')
+        self.close_weakrefs([self.pds])
         if self.context != NULL:
             rc = v.ibv_close_device(self.context)
             if rc != 0:
@@ -131,9 +136,15 @@ cdef class Context(PyverbsObject):
         rc = v.ibv_query_gid(self.context, port_num, index, &gid.gid)
         if rc != 0:
             raise PyverbsRDMAError('Failed to query gid {idx} of port {port}'.
-								   format(idx=index, port=port_num))
+                                                                   format(idx=index, port=port_num))
         return gid
 
+    cdef add_ref(self, obj):
+        if isinstance(obj, PD):
+            self.pds.add(obj)
+        else:
+            raise PyverbsError('Unrecognized object type')
+
 
 cdef class DeviceAttr(PyverbsObject):
     """

pyverbs/libibverbs.pxd

@@ -24,13 +24,13 @@ cdef extern from 'infiniband/verbs.h':
 
     cdef struct ibv_device_attr:
         char            *fw_ver
-        unsigned long   node_guid;
-        unsigned long   sys_image_guid;
-        unsigned long   max_mr_size;
-        unsigned long   page_size_cap;
-        unsigned int    vendor_id;
-        unsigned int    vendor_part_id;
-        unsigned int    hw_ver;
+        unsigned long   node_guid
+        unsigned long   sys_image_guid
+        unsigned long   max_mr_size
+        unsigned long   page_size_cap
+        unsigned int    vendor_id
+        unsigned int    vendor_part_id
+        unsigned int    hw_ver
         unsigned int    max_qp
         unsigned int    max_qp_wr
         unsigned int    device_cap_flags
@@ -46,29 +46,35 @@ cdef extern from 'infiniband/verbs.h':
         unsigned int    max_qp_init_rd_atom
         unsigned int    max_ee_init_rd_atom
         ibv_atomic_cap  atomic_cap
-        unsigned int    max_ee;
-        unsigned int    max_rdd;
-        unsigned int    max_mw;
-        unsigned int    max_raw_ipv6_qp;
-        unsigned int    max_raw_ethy_qp;
-        unsigned int    max_mcast_grp;
-        unsigned int    max_mcast_qp_attach;
-        unsigned int    max_total_mcast_qp_attach;
-        unsigned int    max_ah;
-        unsigned int    max_fmr;
-        unsigned int    max_map_per_fmr;
-        unsigned int    max_srq;
-        unsigned int    max_srq_wr;
-        unsigned int    max_srq_sge;
-        unsigned int    max_pkeys;
-        unsigned int    local_ca_ack_delay;
-        unsigned int    phys_port_cnt;
+        unsigned int    max_ee
+        unsigned int    max_rdd
+        unsigned int    max_mw
+        unsigned int    max_raw_ipv6_qp
+        unsigned int    max_raw_ethy_qp
+        unsigned int    max_mcast_grp
+        unsigned int    max_mcast_qp_attach
+        unsigned int    max_total_mcast_qp_attach
+        unsigned int    max_ah
+        unsigned int    max_fmr
+        unsigned int    max_map_per_fmr
+        unsigned int    max_srq
+        unsigned int    max_srq_wr
+        unsigned int    max_srq_sge
+        unsigned int    max_pkeys
+        unsigned int    local_ca_ack_delay
+        unsigned int    phys_port_cnt
+
+    struct ibv_pd:
+        ibv_context     *context
+        unsigned int    handle
 
     ibv_device **ibv_get_device_list(int *n)
-    void ibv_free_device_list(ibv_device **list);
-    ibv_context *ibv_open_device(ibv_device *device);
+    void ibv_free_device_list(ibv_device **list)
+    ibv_context *ibv_open_device(ibv_device *device)
     int ibv_close_device(ibv_context *context)
     int ibv_query_device(ibv_context *context, ibv_device_attr *device_attr)
     unsigned long ibv_get_device_guid(ibv_device *device)
     int ibv_query_gid(ibv_context *context, unsigned int port_num,
                       int index, ibv_gid *gid)
+    ibv_pd *ibv_alloc_pd(ibv_context *context)
+    int ibv_dealloc_pd(ibv_pd *pd)

pyverbs/pd.pxd

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2019, Mellanox Technologies. All rights reserved.
+from pyverbs.device cimport Context
+cimport pyverbs.libibverbs as v
+from .base cimport PyverbsCM
+
+
+cdef class PD(PyverbsCM):
+    cdef v.ibv_pd *pd
+    cdef Context ctx

pyverbs/pd.pyx

@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2019, Mellanox Technologies. All rights reserved.
+from pyverbs.pyverbs_error import PyverbsRDMAError
+from pyverbs.base import PyverbsRDMAErrno
+
+cdef extern from 'errno.h':
+    int errno
+
+
+cdef class PD(PyverbsCM):
+    def __cinit__(self, Context context not None):
+        """
+        Initializes a PD object. A reference for the creating Context is kept
+        so that Python's GC will destroy the objects in the right order.
+        :param context: The Context object creating the PD
+        :return: The newly created PD on success
+        """
+        self.pd = v.ibv_alloc_pd(<v.ibv_context*>context.context)
+        if self.pd == NULL:
+            raise PyverbsRDMAErrno('Failed to allocate PD', errno)
+        self.ctx = context
+        context.add_ref(self)
+        self.logger.debug('PD: Allocated ibv_pd')
+
+    def __dealloc__(self):
+        """
+        Closes the inner PD.
+        :return: None
+        """
+        self.close()
+
+    cpdef close(self):
+        """
+        Closes the underlying C object of the PD.
+        PD may be deleted directly or indirectly by closing its context, which
+        leaves the Python PD object without the underlying C object, so during
+        destruction, need to check whether or not the C object exists.
+        :return: None
+        """
+        self.logger.debug('Closing PD')
+        if self.pd != NULL:
+            rc = v.ibv_dealloc_pd(self.pd)
+            if rc != 0:
+                raise PyverbsRDMAErrno('Failed to dealloc PD')
+            self.pd = NULL
+            self.ctx = None