From 53af5139c64e593061e338c1a898b9f2d11018b1 Mon Sep 17 00:00:00 2001 From: Noa Osherovich Date: Tue, 12 Feb 2019 11:25:27 +0200 Subject: [PATCH] pyverbs: Introduce PD class This patch introduces PD object which represents ibv_pd. This commit also introduces the usage of weak references. In Python, a weak reference is a reference to an object not strong enough to keep it from being garbage-collected. In Pyverbs, it is used to allow an object to keep track of the objects created using it, so if an object is being destroyed (e.g. explicitly by the user), it can iterate over the dependent objects and destroy the underlying C objects. E.g.: A user opens a device and creates a PD. If the Context object is being destroyed before the PD object, the kernel will return EBUSY. To avoid that, the Context object will hold a weakref to the PD and destroy the C PD before closing itself. This renders the Python PD object useless, but it is a user's choice that we are not blocking. In order to provide a clean teardown, Pyverbs Context is also a context manager, which means itshouldbe used within a 'with' block. This way, when the block is over, teardown will be executed properly. The same applies for all other relevant pyverbs' classes. A PyverbsCM class was added as an alternative base objects to inherit from. It inherits from PyverbsObject and adds the __enter__ and __exit__ functions needed for a Python context manager. Signed-off-by: Noa Osherovich --- pyverbs/CMakeLists.txt | 7 ++--- pyverbs/base.pxd | 6 ++++- pyverbs/base.pyx | 41 ++++++++++++++++++++++++++++- pyverbs/device.pxd | 7 ++--- pyverbs/device.pyx | 21 +++++++++++---- pyverbs/libibverbs.pxd | 58 +++++++++++++++++++++++------------------- pyverbs/pd.pxd | 10 ++++++++ pyverbs/pd.pyx | 46 +++++++++++++++++++++++++++++++++ 8 files changed, 157 insertions(+), 39 deletions(-) create mode 100644 pyverbs/pd.pxd create mode 100644 pyverbs/pd.pyx diff --git a/pyverbs/CMakeLists.txt b/pyverbs/CMakeLists.txt index 8d1f80307..794edee1f 100644 --- a/pyverbs/CMakeLists.txt +++ b/pyverbs/CMakeLists.txt @@ -1,11 +1,12 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) -# Copyright (c) 2018, Mellanox Technologies. All rights reserved. See COPYING file +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file rdma_cython_module(pyverbs - enums.pyx + addr.pyx base.pyx device.pyx - addr.pyx + enums.pyx + pd.pyx ) rdma_python_module(pyverbs diff --git a/pyverbs/base.pxd b/pyverbs/base.pxd index fa5e0dad3..fa661edb5 100644 --- a/pyverbs/base.pxd +++ b/pyverbs/base.pxd @@ -1,5 +1,9 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) -# Copyright (c) 2018, Mellanox Technologies. All rights reserved. +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. cdef class PyverbsObject(object): + cdef object __weakref__ cdef object logger + +cdef class PyverbsCM(PyverbsObject): + cpdef close(self) diff --git a/pyverbs/base.pyx b/pyverbs/base.pyx index 6bcebd095..deee520b1 100644 --- a/pyverbs/base.pyx +++ b/pyverbs/base.pyx @@ -1,5 +1,5 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) -# Copyright (c) 2018, Mellanox Technologies. All rights reserved. +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. import logging from pyverbs.pyverbs_error import PyverbsRDMAError @@ -22,3 +22,42 @@ cdef class PyverbsObject(object): def set_log_level(self, val): self.logger.setLevel(val) + + def close_weakrefs(self, iterables): + """ + For each iterable element of iterables, pop each element and + call its close() method. This method is used when an object is being + closed while other objects still hold C references to it; the object + holds weakrefs to such other object, and closes them before trying to + teardown the C resources. + :param iterables: an array of WeakSets + :return: None + """ + # None elements can be present if an object's close() was called more + # than once (e.g. GC and by another object) + for it in iterables: + if it is None: + continue + while True: + try: + tmp = it.pop() + tmp.close() + except KeyError: # popping an empty set + break + + +cdef class PyverbsCM(PyverbsObject): + """ + This is a base class for pyverbs' context manager objects. It includes + __enter__ and __exit__ functions. + close() is also declared but it should be overridden by each inheriting + class. + """ + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return self.close() + + cpdef close(self): + pass diff --git a/pyverbs/device.pxd b/pyverbs/device.pxd index 7a6489da1..5f3ba4dfc 100644 --- a/pyverbs/device.pxd +++ b/pyverbs/device.pxd @@ -1,14 +1,15 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) # Copyright (c) 2018, Mellanox Technologies. All rights reserved. See COPYING file -from .base cimport PyverbsObject +from .base cimport PyverbsObject, PyverbsCM cimport pyverbs.libibverbs as v -cdef class Context(PyverbsObject): +cdef class Context(PyverbsCM): cdef v.ibv_context *context cdef object name - cpdef close(self) + cdef add_ref(self, obj) + cdef object pds cdef class DeviceAttr(PyverbsObject): cdef v.ibv_device_attr dev_attr diff --git a/pyverbs/device.pyx b/pyverbs/device.pyx index 0264a4f0b..000fb53ff 100644 --- a/pyverbs/device.pyx +++ b/pyverbs/device.pyx @@ -6,11 +6,14 @@ Device module introduces the Context and DeviceAttr class. It allows user to open an IB device (using Context(name=) and query it, which returns a DeviceAttr object. """ -from .pyverbs_error import PyverbsRDMAError +import weakref + +from .pyverbs_error import PyverbsRDMAError, PyverbsError from .pyverbs_error import PyverbsUserError from pyverbs.base import PyverbsRDMAErrno cimport pyverbs.libibverbs as v from pyverbs.addr cimport GID +from pyverbs.pd cimport PD cdef extern from 'errno.h': int errno @@ -55,10 +58,9 @@ class Device(PyverbsObject): guid=guid_to_hex(self.guid)) -cdef class Context(PyverbsObject): +cdef class Context(PyverbsCM): """ - Context class represents the C ibv_context. It currently allows only - querying the underlying device. + Context class represents the C ibv_context. """ def __cinit__(self, **kwargs): """ @@ -71,6 +73,8 @@ cdef class Context(PyverbsObject): """ cdef int count cdef v.ibv_device **dev_list + + self.pds = weakref.WeakSet() dev_name = kwargs.get('name') if dev_name is not None: @@ -106,6 +110,7 @@ cdef class Context(PyverbsObject): cpdef close(self): self.logger.debug('Closing Context') + self.close_weakrefs([self.pds]) if self.context != NULL: rc = v.ibv_close_device(self.context) if rc != 0: @@ -131,9 +136,15 @@ cdef class Context(PyverbsObject): rc = v.ibv_query_gid(self.context, port_num, index, &gid.gid) if rc != 0: raise PyverbsRDMAError('Failed to query gid {idx} of port {port}'. - format(idx=index, port=port_num)) + format(idx=index, port=port_num)) return gid + cdef add_ref(self, obj): + if isinstance(obj, PD): + self.pds.add(obj) + else: + raise PyverbsError('Unrecognized object type') + cdef class DeviceAttr(PyverbsObject): """ diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index f212078f9..d625ddf72 100644 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -24,13 +24,13 @@ cdef extern from 'infiniband/verbs.h': cdef struct ibv_device_attr: char *fw_ver - unsigned long node_guid; - unsigned long sys_image_guid; - unsigned long max_mr_size; - unsigned long page_size_cap; - unsigned int vendor_id; - unsigned int vendor_part_id; - unsigned int hw_ver; + unsigned long node_guid + unsigned long sys_image_guid + unsigned long max_mr_size + unsigned long page_size_cap + unsigned int vendor_id + unsigned int vendor_part_id + unsigned int hw_ver unsigned int max_qp unsigned int max_qp_wr unsigned int device_cap_flags @@ -46,29 +46,35 @@ cdef extern from 'infiniband/verbs.h': unsigned int max_qp_init_rd_atom unsigned int max_ee_init_rd_atom ibv_atomic_cap atomic_cap - unsigned int max_ee; - unsigned int max_rdd; - unsigned int max_mw; - unsigned int max_raw_ipv6_qp; - unsigned int max_raw_ethy_qp; - unsigned int max_mcast_grp; - unsigned int max_mcast_qp_attach; - unsigned int max_total_mcast_qp_attach; - unsigned int max_ah; - unsigned int max_fmr; - unsigned int max_map_per_fmr; - unsigned int max_srq; - unsigned int max_srq_wr; - unsigned int max_srq_sge; - unsigned int max_pkeys; - unsigned int local_ca_ack_delay; - unsigned int phys_port_cnt; + unsigned int max_ee + unsigned int max_rdd + unsigned int max_mw + unsigned int max_raw_ipv6_qp + unsigned int max_raw_ethy_qp + unsigned int max_mcast_grp + unsigned int max_mcast_qp_attach + unsigned int max_total_mcast_qp_attach + unsigned int max_ah + unsigned int max_fmr + unsigned int max_map_per_fmr + unsigned int max_srq + unsigned int max_srq_wr + unsigned int max_srq_sge + unsigned int max_pkeys + unsigned int local_ca_ack_delay + unsigned int phys_port_cnt + + struct ibv_pd: + ibv_context *context + unsigned int handle ibv_device **ibv_get_device_list(int *n) - void ibv_free_device_list(ibv_device **list); - ibv_context *ibv_open_device(ibv_device *device); + void ibv_free_device_list(ibv_device **list) + ibv_context *ibv_open_device(ibv_device *device) int ibv_close_device(ibv_context *context) int ibv_query_device(ibv_context *context, ibv_device_attr *device_attr) unsigned long ibv_get_device_guid(ibv_device *device) int ibv_query_gid(ibv_context *context, unsigned int port_num, int index, ibv_gid *gid) + ibv_pd *ibv_alloc_pd(ibv_context *context) + int ibv_dealloc_pd(ibv_pd *pd) diff --git a/pyverbs/pd.pxd b/pyverbs/pd.pxd new file mode 100644 index 000000000..ec25ec884 --- /dev/null +++ b/pyverbs/pd.pxd @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. +from pyverbs.device cimport Context +cimport pyverbs.libibverbs as v +from .base cimport PyverbsCM + + +cdef class PD(PyverbsCM): + cdef v.ibv_pd *pd + cdef Context ctx diff --git a/pyverbs/pd.pyx b/pyverbs/pd.pyx new file mode 100644 index 000000000..9975f906b --- /dev/null +++ b/pyverbs/pd.pyx @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. +from pyverbs.pyverbs_error import PyverbsRDMAError +from pyverbs.base import PyverbsRDMAErrno + +cdef extern from 'errno.h': + int errno + + +cdef class PD(PyverbsCM): + def __cinit__(self, Context context not None): + """ + Initializes a PD object. A reference for the creating Context is kept + so that Python's GC will destroy the objects in the right order. + :param context: The Context object creating the PD + :return: The newly created PD on success + """ + self.pd = v.ibv_alloc_pd(context.context) + if self.pd == NULL: + raise PyverbsRDMAErrno('Failed to allocate PD', errno) + self.ctx = context + context.add_ref(self) + self.logger.debug('PD: Allocated ibv_pd') + + def __dealloc__(self): + """ + Closes the inner PD. + :return: None + """ + self.close() + + cpdef close(self): + """ + Closes the underlying C object of the PD. + PD may be deleted directly or indirectly by closing its context, which + leaves the Python PD object without the underlying C object, so during + destruction, need to check whether or not the C object exists. + :return: None + """ + self.logger.debug('Closing PD') + if self.pd != NULL: + rc = v.ibv_dealloc_pd(self.pd) + if rc != 0: + raise PyverbsRDMAErrno('Failed to dealloc PD') + self.pd = NULL + self.ctx = None