From 61ad2917b7c06537150bf2e124b708ff53e9b99f Mon Sep 17 00:00:00 2001 From: Itay Aveksis Date: Thu, 21 May 2020 00:32:59 +0300 Subject: [PATCH] pyverbs: Add support for implicit ODP MR The implicit ODP MR covers the entire memory space of a process. Applications using implicit ODP can use the entire address range when writing to the MR, instead of being limited to the address space registered to the MR. This also saves actual pages registration so that application control path is faster. Signed-off-by: Itay Aveksis Signed-off-by: Edward Srouji --- pyverbs/mr.pyx | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pyverbs/mr.pyx b/pyverbs/mr.pyx index da566cbc4..bba6abe27 100644 --- a/pyverbs/mr.pyx +++ b/pyverbs/mr.pyx @@ -7,11 +7,11 @@ import logging from posix.mman cimport mmap, munmap, MAP_PRIVATE, PROT_READ, PROT_WRITE, \ MAP_ANONYMOUS, MAP_HUGETLB from pyverbs.pyverbs_error import PyverbsError, PyverbsRDMAError +from libc.stdint cimport uintptr_t, SIZE_MAX from pyverbs.base import PyverbsRDMAErrno from posix.stdlib cimport posix_memalign from libc.string cimport memcpy, memset cimport pyverbs.libibverbs_enums as e -from libc.stdint cimport uintptr_t from pyverbs.device cimport DM from libc.stdlib cimport free from .pd cimport PD @@ -27,16 +27,18 @@ cdef class MR(PyverbsCM): MR class represents ibv_mr. Buffer allocation in done in the c'tor. Freeing it is done in close(). """ - def __init__(self, PD pd not None, length=0, access=0, address=None, **kwargs): + def __init__(self, PD pd not None, length=0, access=0, address=None, + implicit=False, **kwargs): """ Allocate a user-level buffer of length and register a Memory Region of the given length and access flags. :param pd: A PD object - :param length: Length in bytes + :param length: Length (in bytes) of MR's buffer. :param access: Access flags, see ibv_access_flags enum :param address: Memory address to register (Optional). If it's not provided, a memory will be allocated in the class initialization. + :param implicit: Implicit the MR address. :param kwargs: Arguments: * *handle* A valid kernel handle for a MR object in the given PD. @@ -48,10 +50,6 @@ cdef class MR(PyverbsCM): if self.mr != NULL: return self.is_huge = True if access & e.IBV_ACCESS_HUGETLB else False - # We want to enable registering an MR of size 0 but this fails with a - # buffer of size 0, so in this case lets increase the buffer - if length == 0: - length = 10 if address: self.is_user_addr = True # uintptr_t is guaranteed to be large enough to hold any pointer. @@ -70,7 +68,7 @@ cdef class MR(PyverbsCM): return # Allocate a buffer - if not address: + if not address and length > 0: if self.is_huge: # Rounding up to multiple of HUGE_PAGE_SIZE self.mmap_length = length + (HUGE_PAGE_SIZE - length % HUGE_PAGE_SIZE) \ @@ -86,7 +84,10 @@ cdef class MR(PyverbsCM): raise PyverbsError('Failed to allocate MR buffer of size {l}'. format(l=length)) memset(self.buf, 0, length) - self.mr = v.ibv_reg_mr(pd.pd, self.buf, length, access) + if implicit: + self.mr = v.ibv_reg_mr(pd.pd, NULL, SIZE_MAX, access) + else: + self.mr = v.ibv_reg_mr(pd.pd, self.buf, length, access) if self.mr == NULL: raise PyverbsRDMAErrno('Failed to register a MR. length: {l}, access flags: {a}'. format(l=length, a=access))