diff --git a/pyverbs/CMakeLists.txt b/pyverbs/CMakeLists.txt index 8d1f80307..794edee1f 100644 --- a/pyverbs/CMakeLists.txt +++ b/pyverbs/CMakeLists.txt @@ -1,11 +1,12 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) -# Copyright (c) 2018, Mellanox Technologies. All rights reserved. See COPYING file +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file rdma_cython_module(pyverbs - enums.pyx + addr.pyx base.pyx device.pyx - addr.pyx + enums.pyx + pd.pyx ) rdma_python_module(pyverbs diff --git a/pyverbs/base.pxd b/pyverbs/base.pxd index fa5e0dad3..fa661edb5 100644 --- a/pyverbs/base.pxd +++ b/pyverbs/base.pxd @@ -1,5 +1,9 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) -# Copyright (c) 2018, Mellanox Technologies. All rights reserved. +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. cdef class PyverbsObject(object): + cdef object __weakref__ cdef object logger + +cdef class PyverbsCM(PyverbsObject): + cpdef close(self) diff --git a/pyverbs/base.pyx b/pyverbs/base.pyx index 6bcebd095..deee520b1 100644 --- a/pyverbs/base.pyx +++ b/pyverbs/base.pyx @@ -1,5 +1,5 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) -# Copyright (c) 2018, Mellanox Technologies. All rights reserved. +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. import logging from pyverbs.pyverbs_error import PyverbsRDMAError @@ -22,3 +22,42 @@ cdef class PyverbsObject(object): def set_log_level(self, val): self.logger.setLevel(val) + + def close_weakrefs(self, iterables): + """ + For each iterable element of iterables, pop each element and + call its close() method. This method is used when an object is being + closed while other objects still hold C references to it; the object + holds weakrefs to such other object, and closes them before trying to + teardown the C resources. + :param iterables: an array of WeakSets + :return: None + """ + # None elements can be present if an object's close() was called more + # than once (e.g. GC and by another object) + for it in iterables: + if it is None: + continue + while True: + try: + tmp = it.pop() + tmp.close() + except KeyError: # popping an empty set + break + + +cdef class PyverbsCM(PyverbsObject): + """ + This is a base class for pyverbs' context manager objects. It includes + __enter__ and __exit__ functions. + close() is also declared but it should be overridden by each inheriting + class. + """ + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + return self.close() + + cpdef close(self): + pass diff --git a/pyverbs/device.pxd b/pyverbs/device.pxd index 7a6489da1..5f3ba4dfc 100644 --- a/pyverbs/device.pxd +++ b/pyverbs/device.pxd @@ -1,14 +1,15 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) # Copyright (c) 2018, Mellanox Technologies. All rights reserved. See COPYING file -from .base cimport PyverbsObject +from .base cimport PyverbsObject, PyverbsCM cimport pyverbs.libibverbs as v -cdef class Context(PyverbsObject): +cdef class Context(PyverbsCM): cdef v.ibv_context *context cdef object name - cpdef close(self) + cdef add_ref(self, obj) + cdef object pds cdef class DeviceAttr(PyverbsObject): cdef v.ibv_device_attr dev_attr diff --git a/pyverbs/device.pyx b/pyverbs/device.pyx index 0264a4f0b..000fb53ff 100644 --- a/pyverbs/device.pyx +++ b/pyverbs/device.pyx @@ -6,11 +6,14 @@ Device module introduces the Context and DeviceAttr class. It allows user to open an IB device (using Context(name=) and query it, which returns a DeviceAttr object. """ -from .pyverbs_error import PyverbsRDMAError +import weakref + +from .pyverbs_error import PyverbsRDMAError, PyverbsError from .pyverbs_error import PyverbsUserError from pyverbs.base import PyverbsRDMAErrno cimport pyverbs.libibverbs as v from pyverbs.addr cimport GID +from pyverbs.pd cimport PD cdef extern from 'errno.h': int errno @@ -55,10 +58,9 @@ class Device(PyverbsObject): guid=guid_to_hex(self.guid)) -cdef class Context(PyverbsObject): +cdef class Context(PyverbsCM): """ - Context class represents the C ibv_context. It currently allows only - querying the underlying device. + Context class represents the C ibv_context. """ def __cinit__(self, **kwargs): """ @@ -71,6 +73,8 @@ cdef class Context(PyverbsObject): """ cdef int count cdef v.ibv_device **dev_list + + self.pds = weakref.WeakSet() dev_name = kwargs.get('name') if dev_name is not None: @@ -106,6 +110,7 @@ cdef class Context(PyverbsObject): cpdef close(self): self.logger.debug('Closing Context') + self.close_weakrefs([self.pds]) if self.context != NULL: rc = v.ibv_close_device(self.context) if rc != 0: @@ -131,9 +136,15 @@ cdef class Context(PyverbsObject): rc = v.ibv_query_gid(self.context, port_num, index, &gid.gid) if rc != 0: raise PyverbsRDMAError('Failed to query gid {idx} of port {port}'. - format(idx=index, port=port_num)) + format(idx=index, port=port_num)) return gid + cdef add_ref(self, obj): + if isinstance(obj, PD): + self.pds.add(obj) + else: + raise PyverbsError('Unrecognized object type') + cdef class DeviceAttr(PyverbsObject): """ diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index f212078f9..d625ddf72 100644 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -24,13 +24,13 @@ cdef extern from 'infiniband/verbs.h': cdef struct ibv_device_attr: char *fw_ver - unsigned long node_guid; - unsigned long sys_image_guid; - unsigned long max_mr_size; - unsigned long page_size_cap; - unsigned int vendor_id; - unsigned int vendor_part_id; - unsigned int hw_ver; + unsigned long node_guid + unsigned long sys_image_guid + unsigned long max_mr_size + unsigned long page_size_cap + unsigned int vendor_id + unsigned int vendor_part_id + unsigned int hw_ver unsigned int max_qp unsigned int max_qp_wr unsigned int device_cap_flags @@ -46,29 +46,35 @@ cdef extern from 'infiniband/verbs.h': unsigned int max_qp_init_rd_atom unsigned int max_ee_init_rd_atom ibv_atomic_cap atomic_cap - unsigned int max_ee; - unsigned int max_rdd; - unsigned int max_mw; - unsigned int max_raw_ipv6_qp; - unsigned int max_raw_ethy_qp; - unsigned int max_mcast_grp; - unsigned int max_mcast_qp_attach; - unsigned int max_total_mcast_qp_attach; - unsigned int max_ah; - unsigned int max_fmr; - unsigned int max_map_per_fmr; - unsigned int max_srq; - unsigned int max_srq_wr; - unsigned int max_srq_sge; - unsigned int max_pkeys; - unsigned int local_ca_ack_delay; - unsigned int phys_port_cnt; + unsigned int max_ee + unsigned int max_rdd + unsigned int max_mw + unsigned int max_raw_ipv6_qp + unsigned int max_raw_ethy_qp + unsigned int max_mcast_grp + unsigned int max_mcast_qp_attach + unsigned int max_total_mcast_qp_attach + unsigned int max_ah + unsigned int max_fmr + unsigned int max_map_per_fmr + unsigned int max_srq + unsigned int max_srq_wr + unsigned int max_srq_sge + unsigned int max_pkeys + unsigned int local_ca_ack_delay + unsigned int phys_port_cnt + + struct ibv_pd: + ibv_context *context + unsigned int handle ibv_device **ibv_get_device_list(int *n) - void ibv_free_device_list(ibv_device **list); - ibv_context *ibv_open_device(ibv_device *device); + void ibv_free_device_list(ibv_device **list) + ibv_context *ibv_open_device(ibv_device *device) int ibv_close_device(ibv_context *context) int ibv_query_device(ibv_context *context, ibv_device_attr *device_attr) unsigned long ibv_get_device_guid(ibv_device *device) int ibv_query_gid(ibv_context *context, unsigned int port_num, int index, ibv_gid *gid) + ibv_pd *ibv_alloc_pd(ibv_context *context) + int ibv_dealloc_pd(ibv_pd *pd) diff --git a/pyverbs/pd.pxd b/pyverbs/pd.pxd new file mode 100644 index 000000000..ec25ec884 --- /dev/null +++ b/pyverbs/pd.pxd @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. +from pyverbs.device cimport Context +cimport pyverbs.libibverbs as v +from .base cimport PyverbsCM + + +cdef class PD(PyverbsCM): + cdef v.ibv_pd *pd + cdef Context ctx diff --git a/pyverbs/pd.pyx b/pyverbs/pd.pyx new file mode 100644 index 000000000..9975f906b --- /dev/null +++ b/pyverbs/pd.pyx @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. +from pyverbs.pyverbs_error import PyverbsRDMAError +from pyverbs.base import PyverbsRDMAErrno + +cdef extern from 'errno.h': + int errno + + +cdef class PD(PyverbsCM): + def __cinit__(self, Context context not None): + """ + Initializes a PD object. A reference for the creating Context is kept + so that Python's GC will destroy the objects in the right order. + :param context: The Context object creating the PD + :return: The newly created PD on success + """ + self.pd = v.ibv_alloc_pd(context.context) + if self.pd == NULL: + raise PyverbsRDMAErrno('Failed to allocate PD', errno) + self.ctx = context + context.add_ref(self) + self.logger.debug('PD: Allocated ibv_pd') + + def __dealloc__(self): + """ + Closes the inner PD. + :return: None + """ + self.close() + + cpdef close(self): + """ + Closes the underlying C object of the PD. + PD may be deleted directly or indirectly by closing its context, which + leaves the Python PD object without the underlying C object, so during + destruction, need to check whether or not the C object exists. + :return: None + """ + self.logger.debug('Closing PD') + if self.pd != NULL: + rc = v.ibv_dealloc_pd(self.pd) + if rc != 0: + raise PyverbsRDMAErrno('Failed to dealloc PD') + self.pd = NULL + self.ctx = None