Skip to content

Commit

Permalink
tests: Add Memory Window tests
Browse files Browse the repository at this point in the history 
Add test cases that create and use Memory Window.
This includes both types of Memory Window and invalidate MW as well.
Some previous tests were adjusted or removed since they are now covered
in the newly added tests.

Signed-off-by: Ido Kalir <idok@mellanox.com>
Signed-off-by: Edward Srouji <edwards@mellanox.com>
  • Loading branch information
@idokalir1990 @EdwardSro
idokalir1990 authored and EdwardSro committed Jul 20, 2020
1 parent c05fd07 commit 7b1ec39
Show file tree
Hide file tree
Showing 5 changed files with 231 additions and 85 deletions.
2 changes: 2 additions & 0 deletions Documentation/pyverbs.md
View file
Expand Up @@ -175,6 +175,8 @@ with d.Context(name='mlx5_0') as ctx:
##### Memory window
The following example shows the equivalent of creating a type 1 memory window.
It includes opening a device and allocating the necessary PD.
The user should unbind or close the memory window before being able to
deregister an MR that the MW is bound to.
```python
import pyverbs.device as d
from pyverbs.pd import PD
Expand Down
248 changes: 193 additions & 55 deletions tests/test_mr.py
View file
Expand Up @@ -7,9 +7,11 @@
import random
import errno

from tests.base import PyverbsAPITestCase, RCResources, RDMATestCase
from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsError
from tests.base import PyverbsAPITestCase
from pyverbs.mr import MR, MW, DMMR
from pyverbs.mr import MR, MW, DMMR, MWBindInfo, MWBind
from pyverbs.qp import QPCap, QPInitAttr, QPAttr, QP
from pyverbs.wr import SendWR
import pyverbs.device as d
from pyverbs.pd import PD
import pyverbs.enums as e
Expand Down Expand Up @@ -150,74 +152,210 @@ def test_buffer(self):
mr.buf


class MWTest(PyverbsAPITestCase):
class MWRC(RCResources):
def __init__(self, dev_name, ib_port, gid_index, mw_type):
"""
Initialize Memory Window resources based on RC resources that include RC
QP.
:param dev_name: Device name to be used
:param ib_port: IB port of the device to use
:param gid_index: Which GID index to use
:param mw_type: The MW type to use
"""
super().__init__(dev_name=dev_name, ib_port=ib_port,
gid_index=gid_index)
self.mw_type = mw_type
access = e.IBV_ACCESS_REMOTE_WRITE | e.IBV_ACCESS_LOCAL_WRITE
self.mw_bind_info = MWBindInfo(self.mr, self.mr.buf, self.msg_size,
access)
self.mw_bind = MWBind(self.mw_bind_info, e.IBV_SEND_SIGNALED)
try:
self.mw = MW(self.pd, self.mw_type)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create MW is not supported')
raise ex

def create_mr(self):
access = e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_MW_BIND
try:
self.mr = MR(self.pd, self.msg_size, access)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Reg MR with MW access is not supported')
raise ex

def create_qp(self):
qp_caps = QPCap(max_recv_wr=self.num_msgs)
qp_init_attr = QPInitAttr(qp_type=e.IBV_QPT_RC, scq=self.cq,
rcq=self.cq, cap=qp_caps)
qp_attr = QPAttr(port_num=self.ib_port)
qp_access = e.IBV_ACCESS_LOCAL_WRITE | e.IBV_ACCESS_REMOTE_WRITE
qp_attr.qp_access_flags = qp_access
try:
self.qp = QP(self.pd, qp_init_attr, qp_attr)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create RC QP is not supported')
raise ex



class MWTest(RDMATestCase):
"""
Test various functionalities of the MW class.
"""
def test_reg_mw_type1(self):
"""
Test ibv_alloc_mw() for type 1 MW
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
try:
with MW(pd, e.IBV_MW_TYPE_1):
pass
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create memory window of type 1 is not supported')
raise ex
def setUp(self):
super().setUp()
self.iters = 10
self.server = None
self.client = None

def test_reg_mw_type2(self):
"""
Test ibv_alloc_mw() for type 2 MW
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
try:
with MW(pd, e.IBV_MW_TYPE_2):
pass
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create memory window of type 2 is not supported')
raise ex
def create_players(self, resource, **resource_arg):
"""
Init memory window tests resources.
:param resource: The RDMA resources to use.
:param resource_arg: Dict of args that specify the resource specific
attributes.
:return: None
"""
self.client = resource(**self.dev_info, **resource_arg)
self.server = resource(**self.dev_info, **resource_arg)
self.client.pre_run(self.server.psn, self.server.qpn)
self.server.pre_run(self.client.psn, self.client.qpn)

def test_dereg_mw_type1(self):
"""
Test ibv_dealloc_mw() for type 1 MW
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
try:
with MW(pd, e.IBV_MW_TYPE_1) as mw:
mw.close()
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create memory window of type 1 is not supported')
raise ex
def tearDown(self):
if self.server:
self.server.mw.close()
if self.client:
self.client.mw.close()
return super().tearDown()

def test_dereg_mw_type2(self):
def bind_mw_type_1(self):
self.server.qp.bind_mw(self.server.mw, self.server.mw_bind)
self.client.qp.bind_mw(self.client.mw, self.client.mw_bind)
# Poll the bind MW action completion.
u.poll_cq(self.server.cq)
u.poll_cq(self.client.cq)
self.server.rkey = self.client.mw.rkey
self.server.remote_addr = self.client.mr.buf
self.client.rkey = self.server.mw.rkey
self.client.remote_addr = self.server.mr.buf

def bind_mw_type_2(self):
client_send_wr = SendWR(opcode=e.IBV_WR_BIND_MW)
client_send_wr.set_bind_wr(self.client.mw, self.client.mw_bind_info)
server_send_wr = SendWR(opcode=e.IBV_WR_BIND_MW)
server_send_wr.set_bind_wr(self.server.mw, self.server.mw_bind_info)
self.server.qp.post_send(server_send_wr)
self.client.qp.post_send(client_send_wr)
# Poll the bind MW WR.
u.poll_cq(self.server.cq)
u.poll_cq(self.client.cq)
self.server.rkey = client_send_wr.rkey
self.server.remote_addr = self.client.mr.buf
self.client.rkey = server_send_wr.rkey
self.client.remote_addr = self.server.mr.buf

def invalidate_mw_type1(self):
"""
Test ibv_dealloc_mw() for type 2 MW
Invalidate the MWs by rebind this MW with zero length.
:return: None
"""
for ctx, attr, attr_ex in self.devices:
with PD(ctx) as pd:
try:
with MW(pd, e.IBV_MW_TYPE_2) as mw:
mw.close()
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
raise unittest.SkipTest('Create memory window of type 2 is not supported')
raise ex
for player in [self.server, self.client]:
mw_bind_info = MWBindInfo(player.mr, player.mr.buf, 0, 0)
mw_bind = MWBind(mw_bind_info, e.IBV_SEND_SIGNALED)
player.qp.bind_mw(player.mw, mw_bind)
# Poll the bound MW action request completion.
u.poll_cq(player.cq)

def invalidate_mw_type2_local(self):
"""
Invalidate the MWs by post invalidation send WR from the local QP.
:return: None
"""
inv_send_wr = SendWR(opcode=e.IBV_WR_LOCAL_INV)
inv_send_wr.imm_data = self.server.rkey
self.client.qp.post_send(inv_send_wr)
inv_send_wr = SendWR(opcode=e.IBV_WR_LOCAL_INV)
inv_send_wr.imm_data = self.client.rkey
self.server.qp.post_send(inv_send_wr)
# Poll the invalidate MW WR.
u.poll_cq(self.server.cq)
u.poll_cq(self.client.cq)

def invalidate_mw_type2_remote(self):
"""
Invalidate the MWs by sending invalidation send WR from the remote QP.
:return: None
"""
server_recv_wr = u.get_recv_wr(self.server)
client_recv_wr = u.get_recv_wr(self.client)
self.server.qp.post_recv(server_recv_wr)
self.client.qp.post_recv(client_recv_wr)
inv_send_wr = SendWR(opcode=e.IBV_WR_SEND_WITH_INV)
inv_send_wr.imm_data = self.client.rkey
self.client.qp.post_send(inv_send_wr)
inv_send_wr = SendWR(opcode=e.IBV_WR_SEND_WITH_INV)
inv_send_wr.imm_data = self.server.rkey
self.server.qp.post_send(inv_send_wr)
# Poll the invalidate MW send WR.
u.poll_cq(self.server.cq)
u.poll_cq(self.client.cq)
# Poll the invalidate MW recv WR.
u.poll_cq(self.server.cq)
u.poll_cq(self.client.cq)

def test_mw_type1(self):
self.create_players(MWRC, mw_type=e.IBV_MW_TYPE_1)
self.bind_mw_type_1()
u.rdma_traffic(self.client, self.server, self.iters, self.gid_index,
self.ib_port, send_op=e.IBV_WR_RDMA_WRITE)

def test_invalidate_mw_type1(self):
self.test_mw_type1()
self.invalidate_mw_type1()
with self.assertRaisesRegex(PyverbsRDMAError, 'Remote access error'):
u.rdma_traffic(self.client, self.server, self.iters, self.gid_index,
self.ib_port, send_op=e.IBV_WR_RDMA_WRITE)

def test_mw_type2(self):
self.create_players(MWRC, mw_type=e.IBV_MW_TYPE_2)
self.bind_mw_type_2()
u.rdma_traffic(self.client, self.server, self.iters, self.gid_index,
self.ib_port, send_op=e.IBV_WR_RDMA_WRITE)

def test_mw_type2_invalidate_local(self):
self.test_mw_type2()
self.invalidate_mw_type2_local()
with self.assertRaisesRegex(PyverbsRDMAError, 'Remote access error'):
u.rdma_traffic(self.client, self.server, self.iters, self.gid_index,
self.ib_port, send_op=e.IBV_WR_RDMA_WRITE)

def test_mw_type2_invalidate_remote(self):
self.test_mw_type2()
self.invalidate_mw_type2_remote()
with self.assertRaisesRegex(PyverbsRDMAError, 'Remote access error'):
u.rdma_traffic(self.client, self.server, self.iters, self.gid_index,
self.ib_port, send_op=e.IBV_WR_RDMA_WRITE)

def test_mw_type2_invalidate_dealloc(self):
self.test_mw_type2()
# Dealloc the MW by closing the pyverbs objects.
self.server.mw.close()
self.client.mw.close()
with self.assertRaisesRegex(PyverbsRDMAError, 'Remote access error'):
u.rdma_traffic(self.client, self.server, self.iters, self.gid_index,
self.ib_port, send_op=e.IBV_WR_RDMA_WRITE)

def test_reg_mw_wrong_type(self):
"""
Verify that trying to create a MW of a wrong type fails
"""
for ctx, attr, attr_ex in self.devices:
with d.Context(name=self.dev_name) as ctx:
with PD(ctx) as pd:
try:
mw_type = random.randint(3, 100)
mw_type = 3
MW(pd, mw_type)
except PyverbsRDMAError as ex:
if ex.error_code == errno.EOPNOTSUPP:
Expand Down
18 changes: 9 additions & 9 deletions tests/test_qpex.py
View file
Expand Up @@ -183,12 +183,12 @@ def create_players(self, qp_type):
def test_qp_ex_ud_send(self):
client, server = self.create_players('ud_send')
u.traffic(client, server, self.iters, self.gid_index, self.ib_port,
is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_SEND)
new_send=True, send_op=e.IBV_QP_EX_WITH_SEND)

def test_qp_ex_rc_send(self):
client, server = self.create_players('rc_send')
u.traffic(client, server, self.iters, self.gid_index, self.ib_port,
is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_SEND)
new_send=True, send_op=e.IBV_QP_EX_WITH_SEND)

def test_qp_ex_xrc_send(self):
client, server = self.create_players('xrc_send')
Expand All @@ -197,12 +197,12 @@ def test_qp_ex_xrc_send(self):
def test_qp_ex_ud_send_imm(self):
client, server = self.create_players('ud_send_imm')
u.traffic(client, server, self.iters, self.gid_index, self.ib_port,
is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_SEND_WITH_IMM)
new_send=True, send_op=e.IBV_QP_EX_WITH_SEND_WITH_IMM)

def test_qp_ex_rc_send_imm(self):
client, server = self.create_players('rc_send_imm')
u.traffic(client, server, self.iters, self.gid_index, self.ib_port,
is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_SEND_WITH_IMM)
new_send=True, send_op=e.IBV_QP_EX_WITH_SEND_WITH_IMM)

def test_qp_ex_xrc_send_imm(self):
client, server = self.create_players('xrc_send_imm')
Expand All @@ -215,7 +215,7 @@ def test_qp_ex_rc_rdma_write(self):
client.raddr = server.mr.buf
server.raddr = client.mr.buf
u.rdma_traffic(client, server, self.iters, self.gid_index, self.ib_port,
is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_RDMA_WRITE)
new_send=True, send_op=e.IBV_QP_EX_WITH_RDMA_WRITE)

def test_qp_ex_rc_rdma_write_imm(self):
client, server = self.create_players('rc_write_imm')
Expand All @@ -224,7 +224,7 @@ def test_qp_ex_rc_rdma_write_imm(self):
client.raddr = server.mr.buf
server.raddr = client.mr.buf
u.traffic(client, server, self.iters, self.gid_index, self.ib_port,
is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM)
new_send=True, send_op=e.IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM)

def test_qp_ex_rc_rdma_read(self):
client, server = self.create_players('rc_read')
Expand All @@ -234,7 +234,7 @@ def test_qp_ex_rc_rdma_read(self):
server.raddr = client.mr.buf
server.mr.write('s' * server.msg_size, server.msg_size)
u.rdma_traffic(client, server, self.iters, self.gid_index, self.ib_port,
is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_RDMA_READ)
new_send=True, send_op=e.IBV_QP_EX_WITH_RDMA_READ)

def test_qp_ex_rc_atomic_cmp_swp(self):
client, server = self.create_players('rc_cmp_swp')
Expand All @@ -246,7 +246,7 @@ def test_qp_ex_rc_atomic_cmp_swp(self):
server.raddr = client.mr.buf
server.mr.write('s' * 8, 8)
u.rdma_traffic(client, server, self.iters, self.gid_index, self.ib_port,
is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP)
new_send=True, send_op=e.IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP)

def test_qp_ex_rc_atomic_fetch_add(self):
client, server = self.create_players('rc_fetch_add')
Expand All @@ -258,7 +258,7 @@ def test_qp_ex_rc_atomic_fetch_add(self):
server.raddr = client.mr.buf
server.mr.write('s' * 8, 8)
u.rdma_traffic(client, server, self.iters, self.gid_index, self.ib_port,
is_cq_ex=False, send_op=e.IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD)
new_send=True, send_op=e.IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD)

def test_qp_ex_rc_bind_mw(self):
"""
Expand Down
2 changes: 1 addition & 1 deletion tests/test_shared_pd.py
View file
Expand Up @@ -92,4 +92,4 @@ def test_imported_rc_ex_rdma_write(self):
client.raddr = server_import.mr.buf
server_import.raddr = client.mr.buf
u.rdma_traffic(client, server_import, self.iters, self.gid_index,
self.ib_port, send_op=e.IBV_QP_EX_WITH_RDMA_WRITE)
self.ib_port, send_op=e.IBV_QP_EX_WITH_RDMA_WRITE, new_send=True)

0 comments on commit 7b1ec39

Please sign in to comment.