From c05fd072e3faf61c0a92d99829bf48478abc9e03 Mon Sep 17 00:00:00 2001 From: Ido Kalir Date: Mon, 25 May 2020 23:48:14 +0300 Subject: [PATCH] pyverbs: Add memory window support Add support to create and use Memory Window. Supports both types of Memory Window (MW type 1 and 2). Signed-off-by: Ido Kalir Signed-off-by: Edward Srouji --- pyverbs/libibverbs.pxd | 15 +++++++----- pyverbs/mr.pxd | 4 ++++ pyverbs/mr.pyx | 52 ++++++++++++++++++++++++++++++++++++++++++ pyverbs/qp.pyx | 15 ++++++++++-- pyverbs/wr.pyx | 25 ++++++++++++++++++-- 5 files changed, 101 insertions(+), 10 deletions(-) diff --git a/pyverbs/libibverbs.pxd b/pyverbs/libibverbs.pxd index 2f3447aa4..2fb1322fb 100755 --- a/pyverbs/libibverbs.pxd +++ b/pyverbs/libibverbs.pxd @@ -146,7 +146,7 @@ cdef extern from 'infiniband/verbs.h': ibv_pd *pd unsigned int rkey unsigned int handle - ibv_mw_type mw_type + ibv_mw_type type cdef struct ibv_alloc_dm_attr: size_t length @@ -303,6 +303,11 @@ cdef extern from 'infiniband/verbs.h': unsigned long length unsigned int mw_access_flags + cdef struct ibv_mw_bind: + uint64_t wr_id + unsigned int send_flags + ibv_mw_bind_info bind_info + cdef struct bind_mw: ibv_mw *mw unsigned int rkey @@ -313,10 +318,6 @@ cdef extern from 'infiniband/verbs.h': unsigned short hdr_sz unsigned short mss - cdef union unnamed: - bind_mw bind_mw - tso tso - cdef struct xrc: unsigned int remote_srqn @@ -333,7 +334,8 @@ cdef extern from 'infiniband/verbs.h': unsigned int send_flags wr wr qp_type qp_type - unnamed unnamed + bind_mw bind_mw + tso tso cdef struct ibv_qp_cap: unsigned int max_send_wr @@ -557,6 +559,7 @@ cdef extern from 'infiniband/verbs.h': int ibv_destroy_qp(ibv_qp *qp) int ibv_post_recv(ibv_qp *qp, ibv_recv_wr *wr, ibv_recv_wr **bad_wr) int ibv_post_send(ibv_qp *qp, ibv_send_wr *wr, ibv_send_wr **bad_wr) + int ibv_bind_mw(ibv_qp *qp, ibv_mw *mw, ibv_mw_bind *mw_bind) ibv_xrcd *ibv_open_xrcd(ibv_context *context, ibv_xrcd_init_attr *xrcd_init_attr) int ibv_close_xrcd(ibv_xrcd *xrcd) diff --git a/pyverbs/mr.pxd b/pyverbs/mr.pxd index 7c3bb8e28..3adb2ad0f 100644 --- a/pyverbs/mr.pxd +++ b/pyverbs/mr.pxd @@ -21,6 +21,10 @@ cdef class MWBindInfo(PyverbsCM): cdef v.ibv_mw_bind_info info cdef object mr +cdef class MWBind(PyverbsCM): + cdef v.ibv_mw_bind mw_bind + cdef object mr + cdef class MW(PyverbsCM): cdef object pd cdef v.ibv_mw *mw diff --git a/pyverbs/mr.pyx b/pyverbs/mr.pyx index 6838e8427..b9afe0cf0 100644 --- a/pyverbs/mr.pyx +++ b/pyverbs/mr.pyx @@ -204,6 +204,39 @@ cdef class MWBindInfo(PyverbsCM): self.info.length = length self.info.mw_access_flags = mw_access_flags + @property + def mw_access_flags(self): + return self.info.mw_access_flags + + @property + def length(self): + return self.info.length + + @property + def addr(self): + return self.info.addr + + def __str__(self): + print_format = '{:22}: {:<20}\n' + return 'MWBindInfo:\n' +\ + print_format.format('Addr', self.info.addr) +\ + print_format.format('Length', self.info.length) +\ + print_format.format('MW access flags', self.info.mw_access_flags) + + +cdef class MWBind(PyverbsCM): + def __init__(self, MWBindInfo info not None,send_flags, wr_id=0): + super().__init__() + self.mw_bind.wr_id = wr_id + self.mw_bind.send_flags = send_flags + self.mw_bind.bind_info = info.info + + def __str__(self): + print_format = '{:22}: {:<20}\n' + return 'MWBind:\n' +\ + print_format.format('WR id', self.mw_bind.wr_id) +\ + print_format.format('Send flags', self.mw_bind.send_flags) + cdef class MW(PyverbsCM): def __init__(self, PD pd not None, v.ibv_mw_type mw_type): @@ -242,6 +275,25 @@ cdef class MW(PyverbsCM): self.mw = NULL self.pd = None + @property + def handle(self): + return self.mw.handle + + @property + def rkey(self): + return self.mw.rkey + + @property + def type(self): + return self.mw.type + + def __str__(self): + print_format = '{:22}: {:<20}\n' + return 'MW:\n' +\ + print_format.format('Rkey', self.mw.rkey) +\ + print_format.format('Handle', self.mw.handle) +\ + print_format.format('MW Type', mwtype2str(self.mw.type)) + cdef class DMMR(MR): def __init__(self, PD pd not None, length, access, DM dm, offset): diff --git a/pyverbs/qp.pyx b/pyverbs/qp.pyx index d8848add1..c2695a526 100755 --- a/pyverbs/qp.pyx +++ b/pyverbs/qp.pyx @@ -8,11 +8,11 @@ import weakref from pyverbs.pyverbs_error import PyverbsUserError, PyverbsError, PyverbsRDMAError from pyverbs.utils import gid_str, qp_type_to_str, qp_state_to_str, mtu_to_str from pyverbs.utils import access_flags_to_str, mig_state_to_str -from pyverbs.base import PyverbsRDMAErrno +from pyverbs.mr cimport MW, MWBindInfo, MWBind from pyverbs.wr cimport RecvWR, SendWR, SGE +from pyverbs.base import PyverbsRDMAErrno from pyverbs.addr cimport AHAttr, GID, AH from pyverbs.base cimport close_weakrefs -from pyverbs.mr cimport MW, MWBindInfo cimport pyverbs.libibverbs_enums as e from pyverbs.addr cimport GlobalRoute from pyverbs.device cimport Context @@ -1188,6 +1188,17 @@ cdef class QP(PyverbsCM): raise PyverbsRDMAError('Failed to query ECE', rc) return ece + def bind_mw(self, MW mw not None, MWBind mw_bind): + """ + Bind Memory window type 1. + :param mw: The memory window to bind. + :param mw_bind: MWBind object, includes the bind attributes. + :return: None + """ + rc = v.ibv_bind_mw(self.qp, mw.mw, &mw_bind.mw_bind) + if rc != 0: + raise PyverbsRDMAError('Failed to Bind MW', rc) + @property def qp_type(self): return self.qp.qp_type diff --git a/pyverbs/wr.pyx b/pyverbs/wr.pyx index 3abd6bf68..52e9d55c4 100644 --- a/pyverbs/wr.pyx +++ b/pyverbs/wr.pyx @@ -2,8 +2,10 @@ # Copyright (c) 2019 Mellanox Technologies Inc. All rights reserved. See COPYING file from pyverbs.pyverbs_error import PyverbsUserError, PyverbsError -from pyverbs.base import PyverbsRDMAErrno +from pyverbs.base import PyverbsRDMAErrno, inc_rkey +from pyverbs.mr cimport MW, MR, MWBindInfo cimport pyverbs.libibverbs_enums as e +cimport pyverbs.libibverbs as v from pyverbs.addr cimport AH from libc.stdlib cimport free, malloc from libc.string cimport memcpy @@ -159,7 +161,9 @@ cdef class SendWR(PyverbsCM): cdef v.ibv_sge *dst super().__init__() - if num_sge < 1 or sg is None: + mw_opcodes = [e.IBV_WR_LOCAL_INV, e.IBV_WR_BIND_MW, + e.IBV_WR_SEND_WITH_INV] + if opcode not in mw_opcodes and (num_sge < 1 or sg is None): raise PyverbsUserError('A WR needs at least one SGE') self.send_wr.sg_list = malloc(num_sge * sizeof(v.ibv_sge)) if self.send_wr.sg_list == NULL: @@ -284,6 +288,23 @@ cdef class SendWR(PyverbsCM): self.send_wr.wr.atomic.compare_add = compare_add self.send_wr.wr.atomic.swap = swap + def set_bind_wr(self, MW mw, MWBindInfo bind_info): + """ + Set the members of the bind_mw struct in the send_wr. + :param mw: The MW to bind. + :param bind_info: MWBindInfo object, includes the bind attributes. + :return: None + """ + self.send_wr.bind_mw.mw = mw.mw + # Create the new key from the MW rkey. + rkey = inc_rkey(mw.rkey) + self.send_wr.bind_mw.rkey = rkey + self.send_wr.bind_mw.bind_info = bind_info.info + + @property + def rkey(self): + return self.send_wr.bind_mw.rkey + def set_qp_type_xrc(self, remote_srqn): """ Set the members of the xrc struct in the send_wr's qp_type union, used