From 8aae7abe241c81ef7d461940d7bb7f2973172b99 Mon Sep 17 00:00:00 2001 From: Ido Kalir Date: Tue, 18 Aug 2020 16:03:05 +0300 Subject: [PATCH] tests: Add mlx5 CQ tests Add tests for mlx5dv CQ, including traffic and some bad creation flows. Signed-off-by: Ido Kalir Signed-off-by: Edward Srouji --- tests/CMakeLists.txt | 1 + tests/base.py | 28 ++++- tests/mlx5_base.py | 6 +- tests/test_mlx5_cq.py | 237 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 269 insertions(+), 3 deletions(-) create mode 100644 tests/test_mlx5_cq.py diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4de98d08a..ce3b5ef25 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -14,6 +14,7 @@ rdma_python_test(tests test_cqex.py test_device.py test_efadv.py + test_mlx5_cq.py test_mlx5_dc.py test_mlx5_lag_affinity.py test_mlx5_pp.py diff --git a/tests/base.py b/tests/base.py index 3eb5f5db9..1ca52f0ce 100644 --- a/tests/base.py +++ b/tests/base.py @@ -104,6 +104,29 @@ def __init__(self, methodName='runTest', dev_name=None, ib_port=None, self.gid_index = gid_index self.pkey_index = pkey_index self.ip_addr = None + self.pre_environment = {} + + def set_env_variable(self, var, value): + """ + Set environment variable. The current value for each variable is stored + and is set back at the end of the test. + :param var: The name of the environment variable + :param value: The requested new value of this environment variable + """ + if var not in self.pre_environment.keys(): + self.pre_environment[var] = os.environ.get(var) + os.environ[var] = value + + def tearDown(self): + """ + Restore the previous environment variables values before ending the test. + """ + for k, v in self.pre_environment.items(): + if v is None: + os.environ.pop(k) + else: + os.environ[k] = v + super().tearDown() def is_eth_and_has_roce_hw_bug(self): """ @@ -256,7 +279,7 @@ class TrafficResources(BaseResources): needed for traffic. """ def __init__(self, dev_name, ib_port, gid_index, with_srq=False, - qp_count=1): + qp_count=1, msg_size=1024): """ Initializes a TrafficResources object with the given values and creates basic RDMA resources. @@ -265,11 +288,12 @@ def __init__(self, dev_name, ib_port, gid_index, with_srq=False, :param gid_index: Which GID index to use :param with_srq: If True, create SRQ and attach to QPs :param qp_count: Number of QPs to create + :param msg_size: Size of resource msg. If None, use 1024 as default. """ super(TrafficResources, self).__init__(dev_name=dev_name, ib_port=ib_port, gid_index=gid_index) - self.msg_size = 1024 + self.msg_size = msg_size self.num_msgs = 1000 self.port_attr = None self.mr = None diff --git a/tests/mlx5_base.py b/tests/mlx5_base.py index 099906f35..a4202bae6 100644 --- a/tests/mlx5_base.py +++ b/tests/mlx5_base.py @@ -18,8 +18,9 @@ class Mlx5DcResources(TrafficResources): def __init__(self, dev_name, ib_port, gid_index, send_ops_flags, - qp_count=1): + qp_count=1, create_flags=0): self.send_ops_flags = send_ops_flags + self.create_flags = create_flags super().__init__(dev_name, ib_port, gid_index, with_srq=True, qp_count=qp_count) @@ -77,7 +78,10 @@ def create_qps(self): try: for _ in range(self.qp_count): comp_mask = dve.MLX5DV_QP_INIT_ATTR_MASK_DC + if self.create_flags: + comp_mask |= dve.MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS attr = Mlx5DVQPInitAttr(comp_mask=comp_mask, + create_flags=self.create_flags, dc_init_attr=Mlx5DVDCInitAttr()) qp = Mlx5QP(self.ctx, qp_init_attr, attr) self.qps.append(qp) diff --git a/tests/test_mlx5_cq.py b/tests/test_mlx5_cq.py new file mode 100644 index 000000000..1f757c273 --- /dev/null +++ b/tests/test_mlx5_cq.py @@ -0,0 +1,237 @@ +import unittest +import errno + +from pyverbs.providers.mlx5.mlx5dv import Mlx5Context, Mlx5DVContextAttr, \ + Mlx5DVCQInitAttr, Mlx5CQ, context_flags_to_str +from pyverbs.pyverbs_error import PyverbsRDMAError, PyverbsUserError +from tests.base import RDMATestCase, RCResources +import pyverbs.providers.mlx5.mlx5_enums as dve +from tests.mlx5_base import Mlx5DcResources +from pyverbs.cq import CqInitAttrEx +import pyverbs.enums as e +import tests.utils as u + + +def create_dv_cq(res): + """ + Create Mlx5 DV CQ. + :param res: An instance of BaseResources. + :return: None + """ + dvcq_init_attr = Mlx5DVCQInitAttr() + if res.cqe_comp_res_format: + dvcq_init_attr.cqe_comp_res_format = res.cqe_comp_res_format + dvcq_init_attr.comp_mask |= dve.MLX5DV_CQ_INIT_ATTR_MASK_COMPRESSED_CQE + if res.flags: + dvcq_init_attr.flags = res.flags + dvcq_init_attr.comp_mask |= dve.MLX5DV_CQ_INIT_ATTR_MASK_FLAGS + if res.cqe_size: + dvcq_init_attr.cqe_size = res.cqe_size + dvcq_init_attr.comp_mask |= dve.MLX5DV_CQ_INIT_ATTR_MASK_CQE_SIZE + try: + res.cq = Mlx5CQ(res.ctx, CqInitAttrEx(), dvcq_init_attr) + except PyverbsRDMAError as ex: + if ex.error_code == errno.EOPNOTSUPP: + raise unittest.SkipTest('Create Mlx5DV CQ is not supported') + raise ex + + +class Mlx5CQRes(RCResources): + def __init__(self, dev_name, ib_port, gid_index, cqe_comp_res_format=None, + flags=None, cqe_size=None, msg_size=1024, requested_dev_cap=None): + """ + Initialize Mlx5 DV CQ resources based on RC resources that include RC + QP. + :param dev_name: Device name to be used + :param ib_port: IB port of the device to use + :param gid_index: Which GID index to use + :param cqe_comp_res_format: Type of compression to use + :param flags: DV CQ specific flags + :param cqe_size: The CQE size + :param msg_size: The resource msg size + :param requested_dev_cap: A necessary device cap. If it's not supported + by the device, the test will be skipped. + """ + self.cqe_comp_res_format = cqe_comp_res_format + self.flags = flags + self.cqe_size = cqe_size + self.requested_dev_cap = requested_dev_cap + super().__init__(dev_name, ib_port, gid_index, msg_size=msg_size) + + def create_context(self): + mlx5dv_attr = Mlx5DVContextAttr() + try: + self.ctx = Mlx5Context(mlx5dv_attr, name=self.dev_name) + except PyverbsUserError as ex: + raise unittest.SkipTest(f'Could not open mlx5 context ({ex})') + except PyverbsRDMAError: + raise unittest.SkipTest('Opening mlx5 context is not supported') + if self.requested_dev_cap: + if not self.ctx.query_mlx5_device().flags & self.requested_dev_cap: + miss_caps = context_flags_to_str(self.requested_dev_cap) + raise unittest.SkipTest(f'Device caps doesn\'t support {miss_caps}') + + def create_cq(self): + create_dv_cq(self) + + +class Mlx5DvCqDcRes(Mlx5DcResources): + def __init__(self, dev_name, ib_port, gid_index, cqe_comp_res_format=None, + flags=None, cqe_size=None, create_flags=None): + """ + Initialize Mlx5 DV CQ resources based on RC resources that include RC + QP. + :param dev_name: Device name to be used + :param ib_port: IB port of the device to use + :param gid_index: Which GID index to use + :param cqe_comp_res_format: Type of compression to use + :param flags: DV CQ specific flags + :param cqe_size: The CQ's CQe size + :param create_flags: DV QP specific flags + """ + self.cqe_comp_res_format = cqe_comp_res_format + self.flags = flags + self.cqe_size = cqe_size + super().__init__(dev_name, ib_port, gid_index, + send_ops_flags=e.IBV_QP_EX_WITH_SEND, + create_flags=create_flags) + + def create_cq(self): + create_dv_cq(self) + + +class DvCqTest(RDMATestCase): + def setUp(self): + super().setUp() + self.iters = 10 + self.server = None + self.client = None + self.traffic_args = None + + def create_players(self, resource, **resource_arg): + """ + Init DV CQ tests resources. + :param resource: The RDMA resources to use. + :param resource_arg: Dict of args that specify the resource specific + attributes. + :return: None + """ + self.client = resource(**self.dev_info, **resource_arg) + self.server = resource(**self.dev_info, **resource_arg) + self.client.pre_run(self.server.psns, self.server.qps_num) + self.server.pre_run(self.client.psns, self.client.qps_num) + if resource == Mlx5DvCqDcRes: + self.client.remote_dct_num = self.server.dct_qp.qp_num + self.server.remote_dct_num = self.client.dct_qp.qp_num + self.traffic_args = {'client': self.client, 'server': self.server, + 'iters': self.iters, 'gid_idx': self.gid_index, + 'port': self.ib_port} + + def test_dv_cq_traffic(self): + """ + Run SEND traffic using DC CQ. + """ + self.create_players(Mlx5CQRes) + u.traffic(**self.traffic_args, is_cq_ex=True) + + def test_dv_cq_compression_flags(self): + """ + Create DV CQ with different types of CQE compression formats. The test + also does bad flow and try to use more than one compression formats. + """ + # Create DV CQ with all legal compression flags. + for comp_type in [dve.MLX5DV_CQE_RES_FORMAT_CSUM_STRIDX, + dve.MLX5DV_CQE_RES_FORMAT_CSUM, + dve.MLX5DV_CQE_RES_FORMAT_HASH]: + self.create_players(Mlx5CQRes, cqe_comp_res_format=comp_type, + requested_dev_cap=dve.MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP) + u.traffic(**self.traffic_args, is_cq_ex=True) + + # Try to create DV CQ with more than one compression flags. + cqe_multi_format = dve.MLX5DV_CQE_RES_FORMAT_HASH | \ + dve.MLX5DV_CQE_RES_FORMAT_CSUM + with self.assertRaises(PyverbsRDMAError) as ex: + self.create_players(Mlx5CQRes, cqe_comp_res_format=cqe_multi_format) + self.assertEqual(ex.exception.error_code, errno.EINVAL) + + def test_dv_cq_padding(self): + """ + Create DV CQ with padding flag. + """ + self.create_players(Mlx5CQRes, cqe_size=128, + flags=dve.MLX5DV_CQ_INIT_ATTR_FLAGS_CQE_PAD, + requested_dev_cap=dve.MLX5DV_CONTEXT_FLAGS_CQE_128B_PAD) + u.traffic(**self.traffic_args, is_cq_ex=True) + + def test_dv_cq_padding_not_aligned_cqe_size(self): + """ + Create DV CQ with padding flag when CQE size is not 128B. The creation + should fail because padding is supported only with CQE size of 128B. + """ + # Padding flag works only when the cqe size is 128. + with self.assertRaises(PyverbsRDMAError) as ex: + self.create_players(Mlx5CQRes, cqe_size=64, + flags=dve.MLX5DV_CQ_INIT_ATTR_FLAGS_CQE_PAD, + requested_dev_cap=dve.MLX5DV_CONTEXT_FLAGS_CQE_128B_PAD) + self.assertEqual(ex.exception.error_code, errno.EINVAL) + + def test_dv_cq_cqe_size_128(self): + """ + Test multiple sizes of msg using CQE size of 128B. + """ + msg_sizes = [60, # Lower than 64B + 70, # In range of 64B - 128B + 140] # Bigger than 128B + for size in msg_sizes: + self.create_players(Mlx5CQRes, cqe_size=128, msg_size=size) + u.traffic(**self.traffic_args, is_cq_ex=True) + + def test_dv_cq_cqe_size_64(self): + """ + Test multiple sizes of msg using CQE size of 64B. + """ + msg_sizes = [16, # Lower than 32B + 60, # In range of 32B - 64B + 70] # Bigger than 64B + for size in msg_sizes: + self.create_players(Mlx5CQRes, cqe_size=64, msg_size=size) + u.traffic(**self.traffic_args, is_cq_ex=True) + + def test_dv_cq_cqe_size_with_bad_size(self): + """ + Create CQ with ilegal cqe_size value. + """ + # Set the CQE size in the CQE creation. + with self.assertRaises(PyverbsRDMAError) as ex: + self.create_players(Mlx5CQRes, cqe_size=100) + self.assertEqual(ex.exception.error_code, errno.EINVAL) + + # Set the CQE size using the environment value. + self.set_env_variable('MLX5_CQE_SIZE', '100') + with self.assertRaises(PyverbsRDMAError) as ex: + self.create_players(Mlx5CQRes) + self.assertEqual(ex.exception.error_code, errno.EINVAL) + + def test_dv_cq_cqe_size_environment_var(self): + """ + Create DV CQs with all the legal cqe_size values using the environment + variable mechanism. + """ + for cqe_size in ['64', '128']: + self.set_env_variable('MLX5_CQE_SIZE', cqe_size) + self.create_players(Mlx5CQRes) + + def test_scatter_to_cqe_control_by_qp(self): + """ + Create QP with specific SCATTER_TO_CQE flags. The test set different + values in the scatter2cqe environment variable and create the QP with + enable/disable flags. The QP should ignore the environment variable + value and behave according to the specific creation flag. + """ + for s2c_env_val in ['0', '1']: + for qp_s2c_value in [dve.MLX5DV_QP_CREATE_DISABLE_SCATTER_TO_CQE, + dve.MLX5DV_QP_CREATE_ALLOW_SCATTER_TO_CQE]: + self.set_env_variable('MLX5_SCATTER_TO_CQE', s2c_env_val) + self.create_players(Mlx5DvCqDcRes, create_flags=qp_s2c_value) + u.traffic(**self.traffic_args, new_send=True, + send_op=e.IBV_QP_EX_WITH_SEND, is_cq_ex=True)