Skip to content

Commit

Permalink
Merge pull request #788 from yishaih/mlx5_misc
Browse files Browse the repository at this point in the history 
mlx5:  Enable SW Steering RX/TX domains
  • Loading branch information
@yishaih
yishaih committed Jul 14, 2020
2 parents 66797c6 + 244015c commit e055d8c
Show file tree
Hide file tree
Showing 5 changed files with 67 additions and 21 deletions.
42 changes: 32 additions & 10 deletions providers/mlx5/dr_devx.c
View file
Expand Up @@ -133,6 +133,7 @@ int dr_devx_query_device(struct ibv_context *ctx, struct dr_devx_caps *caps)
{
uint32_t out[DEVX_ST_SZ_DW(query_hca_cap_out)] = {};
uint32_t in[DEVX_ST_SZ_DW(query_hca_cap_in)] = {};
bool roce;
int err;

DEVX_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
Expand All @@ -151,6 +152,8 @@ int dr_devx_query_device(struct ibv_context *ctx, struct dr_devx_caps *caps)
caps->gvmi = DEVX_GET(query_hca_cap_out, out, capability.cmd_hca_cap.vhca_id);
caps->flex_protocols = DEVX_GET(query_hca_cap_out, out,
capability.cmd_hca_cap.flex_parser_protocols);
roce = DEVX_GET(query_hca_cap_out, out,
capability.cmd_hca_cap.roce);

if (dr_matcher_supp_flex_parser_icmp_v4(caps)) {
caps->flex_parser_id_icmp_dw0 =
Expand Down Expand Up @@ -217,6 +220,21 @@ int dr_devx_query_device(struct ibv_context *ctx, struct dr_devx_caps *caps)
caps->hdr_modify_icm_addr = DEVX_GET64(query_hca_cap_out, out,
capability.device_mem_cap.
header_modify_sw_icm_start_address);
/* RoCE caps */
if (roce) {
DEVX_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
DEVX_SET(query_hca_cap_in, in, op_mod,
MLX5_SET_HCA_CAP_OP_MOD_ROCE |
HCA_CAP_OPMOD_GET_CUR);

err = mlx5dv_devx_general_cmd(ctx, in, sizeof(in), out, sizeof(out));
if (err) {
dr_dbg_ctx(ctx, "Query RoCE capabilities failed %d\n", err);
return err;
}
caps->roce_caps.fl_rc_qp_when_roce_enabled = DEVX_GET(query_hca_cap_out, out,
capability.roce_caps.fl_rc_qp_when_roce_enabled);
}

return 0;
}
Expand Down Expand Up @@ -467,16 +485,20 @@ int dr_devx_modify_qp_init2rtr(struct ibv_context *ctx,
DEVX_SET(qpc, qpc, mtu, attr->mtu);
DEVX_SET(qpc, qpc, log_msg_max, DR_CHUNK_SIZE_MAX - 1);
DEVX_SET(qpc, qpc, remote_qpn, attr->qp_num);
memcpy(DEVX_ADDR_OF(qpc, qpc, primary_address_path.rmac_47_32),
attr->dgid_attr.mac, sizeof(attr->dgid_attr.mac));
memcpy(DEVX_ADDR_OF(qpc, qpc, primary_address_path.rgid_rip),
attr->dgid_attr.gid.raw, sizeof(attr->dgid_attr.gid.raw));
DEVX_SET(qpc, qpc, primary_address_path.src_addr_index,
attr->sgid_index);

if (attr->dgid_attr.roce_ver == MLX5_ROCE_VERSION_2)
DEVX_SET(qpc, qpc, primary_address_path.udp_sport,
DR_DEVX_ICM_UDP_PORT);

if (attr->fl) {
DEVX_SET(qpc, qpc, primary_address_path.fl, attr->fl);
} else {
memcpy(DEVX_ADDR_OF(qpc, qpc, primary_address_path.rmac_47_32),
attr->dgid_attr.mac, sizeof(attr->dgid_attr.mac));
memcpy(DEVX_ADDR_OF(qpc, qpc, primary_address_path.rgid_rip),
attr->dgid_attr.gid.raw, sizeof(attr->dgid_attr.gid.raw));
DEVX_SET(qpc, qpc, primary_address_path.src_addr_index,
attr->sgid_index);
if (attr->dgid_attr.roce_ver == MLX5_ROCE_VERSION_2)
DEVX_SET(qpc, qpc, primary_address_path.udp_sport,
DR_DEVX_ICM_UDP_PORT);
}

DEVX_SET(qpc, qpc, primary_address_path.vhca_port_num, attr->port_num);
DEVX_SET(qpc, qpc, min_rnr_nak, 1);
Expand Down
11 changes: 7 additions & 4 deletions providers/mlx5/dr_domain.c
View file
Expand Up @@ -188,17 +188,20 @@ static int dr_domain_caps_init(struct ibv_context *ctx,
if (ret)
return ret;

/* Non FDB type is supported only over root table */
if (dmn->type != MLX5DV_DR_DOMAIN_TYPE_FDB)
return 0;

ret = dr_devx_query_device(ctx, &dmn->info.caps);
if (ret)
/* Ignore devx query failure to allow steering on root level
* tables in case devx is not supported over mlx5dv_dr API
*/
return 0;

/* Non FDB type is supported over root table or when we can enable
* force-loopback.
*/
if ((dmn->type != MLX5DV_DR_DOMAIN_TYPE_FDB) &&
!dmn->info.caps.roce_caps.fl_rc_qp_when_roce_enabled)
return 0;

ret = dr_domain_query_fdb_caps(ctx, dmn);
if (ret)
return ret;
Expand Down
19 changes: 12 additions & 7 deletions providers/mlx5/dr_send.c
View file
Expand Up @@ -838,17 +838,22 @@ static int dr_prepare_qp_to_rts(struct mlx5dv_dr_domain *dmn)
}

/* RTR */
ret = dr_devx_query_gid(dmn->ctx, port, gid_index, &rtr_attr.dgid_attr);
if (ret) {
dr_dbg(dmn, "can't read sgid of index %d\n", gid_index);
return ret;
}

rtr_attr.mtu = mtu;
rtr_attr.qp_num = dr_qp->obj->object_id;
rtr_attr.min_rnr_timer = 12;
rtr_attr.port_num = port;
rtr_attr.sgid_index = gid_index;

/* Enable force-loopback on the QP */
if (dmn->info.caps.roce_caps.fl_rc_qp_when_roce_enabled) {
rtr_attr.fl = true;
} else {
ret = dr_devx_query_gid(dmn->ctx, port, gid_index, &rtr_attr.dgid_attr);
if (ret) {
dr_dbg(dmn, "can't read sgid of index %d\n", gid_index);
return ret;
}
rtr_attr.sgid_index = gid_index;
}

ret = dr_devx_modify_qp_init2rtr(dmn->ctx, dr_qp->obj, &rtr_attr);
if (ret) {
Expand Down
10 changes: 10 additions & 0 deletions providers/mlx5/mlx5_ifc.h
View file
Expand Up @@ -92,6 +92,14 @@ struct mlx5_ifc_atomic_caps_bits {
u8 reserved_at_2b0[0x550];
};

struct mlx5_ifc_roce_cap_bits {
u8 reserved_0[0x6];
u8 fl_rc_qp_when_roce_enabled[0x1];
u8 reserved_at_7[0x19];

u8 reserved_at_20[0x7e0];
};

struct mlx5_ifc_flow_table_context_bits {
u8 reformat_en[0x1];
u8 decap_en[0x1];
Expand Down Expand Up @@ -1034,6 +1042,7 @@ union mlx5_ifc_hca_cap_union_bits {
struct mlx5_ifc_flow_table_eswitch_cap_bits flow_table_eswitch_cap;
struct mlx5_ifc_device_mem_cap_bits device_mem_cap;
struct mlx5_ifc_odp_cap_bits odp_cap;
struct mlx5_ifc_roce_cap_bits roce_caps;
u8 reserved_at_0[0x8000];
};

Expand Down Expand Up @@ -1069,6 +1078,7 @@ enum mlx5_cap_type {

enum {
MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE = 0x0 << 1,
MLX5_SET_HCA_CAP_OP_MOD_ROCE = 0x4 << 1,
MLX5_SET_HCA_CAP_OP_MOD_NIC_FLOW_TABLE = 0x7 << 1,
MLX5_SET_HCA_CAP_OP_MOD_ESW_FLOW_TABLE = 0x8 << 1,
MLX5_SET_HCA_CAP_OP_MOD_DEVICE_MEMORY = 0xf << 1,
Expand Down
6 changes: 6 additions & 0 deletions providers/mlx5/mlx5dv_dr.h
View file
Expand Up @@ -575,6 +575,10 @@ struct dr_devx_vport_cap {
uint64_t icm_address_tx;
};

struct dr_devx_roce_cap {
bool fl_rc_qp_when_roce_enabled;
};

struct dr_devx_caps {
uint16_t gvmi;
uint64_t nic_rx_drop_address;
Expand All @@ -596,6 +600,7 @@ struct dr_devx_caps {
bool fdb_sw_owner;
uint32_t num_vports;
struct dr_devx_vport_cap *vports_caps;
struct dr_devx_roce_cap roce_caps;
};

struct dr_domain_rx_tx {
Expand Down Expand Up @@ -913,6 +918,7 @@ struct dr_devx_qp_rtr_attr {
uint16_t port_num;
uint8_t min_rnr_timer;
uint8_t sgid_index;
bool fl;
};

int dr_devx_modify_qp_init2rtr(struct ibv_context *ctx,
Expand Down

0 comments on commit e055d8c

Please sign in to comment.