From 3a44b4c2ae4935535591c74a2506971dda44fccf Mon Sep 17 00:00:00 2001 From: Noa Osherovich Date: Mon, 18 Feb 2019 15:47:42 +0200 Subject: [PATCH] pyverbs: Introducing MR object MR represnts ibv_mr. Instead of allocating a buffer prior to calling ibv_reg_mr, the user provides the MR class the required size and flags, and pyverbs allocates the needed buffer. The MR class exposes read() and write() methods to users to allow setting and reading the underlying memory buffer. Signed-off-by: Noa Osherovich --- pyverbs/CMakeLists.txt | 1 + pyverbs/libibverbs.pxd | 11 +++++ pyverbs/mr.pxd | 12 +++++ pyverbs/mr.pyx | 109 +++++++++++++++++++++++++++++++++++++++++ pyverbs/pd.pxd | 2 + pyverbs/pd.pyx | 13 ++++- 6 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 pyverbs/mr.pxd create mode 100644 pyverbs/mr.pyx diff --git a/pyverbs/CMakeLists.txt b/pyverbs/CMakeLists.txt index c1e7dceb8..b660b46f1 100644 --- a/pyverbs/CMakeLists.txt +++ b/pyverbs/CMakeLists.txt @@ -6,6 +6,7 @@ rdma_cython_module(pyverbs base.pyx device.pyx enums.pyx + mr.pyx pd.pyx ) diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index d625ddf72..5409ff04a 100644 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -68,6 +68,15 @@ cdef extern from 'infiniband/verbs.h': ibv_context *context unsigned int handle + cdef struct ibv_mr: + ibv_context *context + ibv_pd *pd + void *addr + size_t length + unsigned int handle + unsigned int lkey + unsigned int rkey + ibv_device **ibv_get_device_list(int *n) void ibv_free_device_list(ibv_device **list) ibv_context *ibv_open_device(ibv_device *device) @@ -78,3 +87,5 @@ cdef extern from 'infiniband/verbs.h': int index, ibv_gid *gid) ibv_pd *ibv_alloc_pd(ibv_context *context) int ibv_dealloc_pd(ibv_pd *pd) + ibv_mr *ibv_reg_mr(ibv_pd *pd, void *addr, size_t length, int access) + int ibv_dereg_mr(ibv_mr *mr) diff --git a/pyverbs/mr.pxd b/pyverbs/mr.pxd new file mode 100644 index 000000000..92578021c --- /dev/null +++ b/pyverbs/mr.pxd @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file + +from pyverbs.base cimport PyverbsCM +from . cimport libibverbs as v + + +cdef class MR(PyverbsCM): + cdef object pd + cdef v.ibv_mr *mr + cdef void *buf + cpdef read(self, length, offset) diff --git a/pyverbs/mr.pyx b/pyverbs/mr.pyx new file mode 100644 index 000000000..31949e739 --- /dev/null +++ b/pyverbs/mr.pyx @@ -0,0 +1,109 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2019, Mellanox Technologies. All rights reserved. See COPYING file + +from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError +from pyverbs.base import PyverbsRDMAErrno +from .pd cimport PD +import resource + +cdef extern from 'stdlib.h': + int posix_memalign(void **memptr, size_t alignment, size_t size) +cdef extern from 'stdlib.h': + void free(void *ptr) +cdef extern from 'string.h': + void *memcpy(void *dest, const void *src, size_t n) + void *memset(void *s, int c, size_t n) +cdef extern from 'stdint.h': + ctypedef int uintptr_t + + +cdef class MR(PyverbsCM): + """ + MR class represents ibv_mr. Buffer allocation in done in the c'tor. Freeing + it is done in close(). + """ + def __cinit__(self, PD pd not None, length, access): + """ + Allocate a user-level buffer of length and register a Memory + Region of the given length and access flags. + :param pd: A PD object + :param length: Length in bytes + :param access: Access flags, see ibv_access_flags enum + :return: The newly created MR on success + """ + #We want to enable registering an MR of size 0 but this fails with a + #buffer of size 0, so in this case lets increase the buffer + if length == 0: + length = 10 + rc = posix_memalign(&self.buf, resource.getpagesize(), length) + if rc: + raise PyverbsRDMAError('Failed to allocate MR buffer of size {l}'. + format(l=length)) + memset(self.buf, 0, length) + self.mr = v.ibv_reg_mr(pd.pd, self.buf, length, access) + if self.mr == NULL: + raise PyverbsRDMAErrno('Failed to register a MR. length: {l}, access flags: {a}'. + format(l=length, a=access)) + self.pd = pd + pd.add_ref(self) + self.logger.debug('Registered ibv_mr. Length: {l}, access flags {a}'. + format(l=length, a=access)) + + def __dealloc__(self): + self.close() + + cpdef close(self): + """ + Closes the underlying C object of the MR and frees the memory allocated. + MR may be deleted directly or indirectly by closing its context, which + leaves the Python PD object without the underlying C object, so during + destruction, need to check whether or not the C object exists. + :return: None + """ + self.logger.debug('Closing MR') + if self.mr != NULL: + rc = v.ibv_dereg_mr(self.mr) + if rc != 0: + raise PyverbsRDMAErrno('Failed to dereg MR') + self.mr = NULL + self.pd = None + free(self.buf) + self.buf = NULL + + def write(self, data, length): + """ + Write user data to the MR's buffer using memcpy + :param data: User data to write + :param length: Length of the data to write + :return: None + """ + # If data is a string, cast it to bytes as Python3 doesn't + # automatically convert it. + if isinstance(data, str): + data = data.encode() + memcpy(self.buf, data, length) + + cpdef read(self, length, offset): + """ + Reads data from the MR's buffer + :param length: Length of data to read + :param offset: Reading offset + :return: The data on the buffer in the requested offset + """ + cdef char *data + cdef int off = offset # we can't use offset in the next line, as it is + # a Python object and not C + data = (self.buf + off) + return data[:length] + + @property + def buf(self): + return self.buf + + @property + def lkey(self): + return self.mr.lkey + + @property + def rkey(self): + return self.mr.rkey diff --git a/pyverbs/pd.pxd b/pyverbs/pd.pxd index ec25ec884..8381662b2 100644 --- a/pyverbs/pd.pxd +++ b/pyverbs/pd.pxd @@ -8,3 +8,5 @@ from .base cimport PyverbsCM cdef class PD(PyverbsCM): cdef v.ibv_pd *pd cdef Context ctx + cdef add_ref(self, obj) + cdef object mrs diff --git a/pyverbs/pd.pyx b/pyverbs/pd.pyx index 9975f906b..a5fc06375 100644 --- a/pyverbs/pd.pyx +++ b/pyverbs/pd.pyx @@ -1,7 +1,10 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) # Copyright (c) 2019, Mellanox Technologies. All rights reserved. -from pyverbs.pyverbs_error import PyverbsRDMAError +import weakref + +from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError from pyverbs.base import PyverbsRDMAErrno +from .mr cimport MR cdef extern from 'errno.h': int errno @@ -21,6 +24,7 @@ cdef class PD(PyverbsCM): self.ctx = context context.add_ref(self) self.logger.debug('PD: Allocated ibv_pd') + self.mrs = weakref.WeakSet() def __dealloc__(self): """ @@ -38,9 +42,16 @@ cdef class PD(PyverbsCM): :return: None """ self.logger.debug('Closing PD') + self.close_weakrefs([self.mrs]) if self.pd != NULL: rc = v.ibv_dealloc_pd(self.pd) if rc != 0: raise PyverbsRDMAErrno('Failed to dealloc PD') self.pd = NULL self.ctx = None + + cdef add_ref(self, obj): + if isinstance(obj, MR): + self.mrs.add(obj) + else: + raise PyverbsError('Unrecognized object type')