From 858451ad3db1ba6e135253c3aed0d0906989befd Mon Sep 17 00:00:00 2001 From: Shun Hao Date: Sun, 28 Jun 2020 11:21:03 +0300 Subject: [PATCH 1/2] mlx5: DR, Support creating FW owned terminating flow table Terminating flow table acts as the final one of a series of steering tables, forwarding all packets to destinations like VPORT/TIR, but cannot target to other flow tables. Terminating flow table only supports FW owner and always-hit match criteria. So must use devx to create terminating FT and its always-hit FT/FTE. This will enable features that require terminating flow table being supported as of sampler that is introduced in the next patch. Signed-off-by: Shun Hao Reviewed-by: Alex Vesker Signed-off-by: Yishai Hadas --- providers/mlx5/dr_devx.c | 222 +++++++++++++++++++++++++++++++++---- providers/mlx5/dr_table.c | 17 ++- providers/mlx5/mlx5.h | 2 + providers/mlx5/mlx5_ifc.h | 112 ++++++++++++++++++- providers/mlx5/mlx5dv_dr.h | 56 +++++++++- providers/mlx5/verbs.c | 8 ++ 6 files changed, 380 insertions(+), 37 deletions(-) diff --git a/providers/mlx5/dr_devx.c b/providers/mlx5/dr_devx.c index cd0f8bbc2..af372301f 100644 --- a/providers/mlx5/dr_devx.c +++ b/providers/mlx5/dr_devx.c @@ -257,41 +257,219 @@ int dr_devx_sync_steering(struct ibv_context *ctx) return err; } -struct mlx5dv_devx_obj *dr_devx_create_flow_table(struct ibv_context *ctx, - uint32_t table_type, - uint64_t icm_addr_rx, - uint64_t icm_addr_tx, - u8 level) +struct mlx5dv_devx_obj * +dr_devx_create_flow_table(struct ibv_context *ctx, + struct dr_devx_flow_table_attr *ft_attr) { uint32_t out[DEVX_ST_SZ_DW(create_flow_table_out)] = {}; uint32_t in[DEVX_ST_SZ_DW(create_flow_table_in)] = {}; void *ft_ctx; DEVX_SET(create_flow_table_in, in, opcode, MLX5_CMD_OP_CREATE_FLOW_TABLE); - DEVX_SET(create_flow_table_in, in, table_type, table_type); + DEVX_SET(create_flow_table_in, in, table_type, ft_attr->type); ft_ctx = DEVX_ADDR_OF(create_flow_table_in, in, flow_table_context); - DEVX_SET(flow_table_context, ft_ctx, sw_owner, 1); - - DEVX_SET(flow_table_context, ft_ctx, level, level); - /* - * icm_addr_0 used for FDB RX / NIC TX / NIC_RX - * icm_addr_1 used for FDB TX - */ - if (table_type == FS_FT_NIC_RX) { - DEVX_SET64(flow_table_context, ft_ctx, sw_owner_icm_root_0, icm_addr_rx); - } else if (table_type == FS_FT_NIC_TX) { - DEVX_SET64(flow_table_context, ft_ctx, sw_owner_icm_root_0, icm_addr_tx); - } else if (table_type == FS_FT_FDB) { - DEVX_SET64(flow_table_context, ft_ctx, sw_owner_icm_root_0, icm_addr_rx); - DEVX_SET64(flow_table_context, ft_ctx, sw_owner_icm_root_1, icm_addr_tx); - } else { - assert(false); + DEVX_SET(flow_table_context, ft_ctx, termination_table, ft_attr->term_tbl); + DEVX_SET(flow_table_context, ft_ctx, sw_owner, ft_attr->sw_owner); + DEVX_SET(flow_table_context, ft_ctx, level, ft_attr->level); + + if (ft_attr->sw_owner) { + /* icm_addr_0 used for FDB RX / NIC TX / NIC_RX + * icm_addr_1 used for FDB TX + */ + if (ft_attr->type == FS_FT_NIC_RX) { + DEVX_SET64(flow_table_context, ft_ctx, + sw_owner_icm_root_0, ft_attr->icm_addr_rx); + } else if (ft_attr->type == FS_FT_NIC_TX) { + DEVX_SET64(flow_table_context, ft_ctx, + sw_owner_icm_root_0, ft_attr->icm_addr_tx); + } else if (ft_attr->type == FS_FT_FDB) { + DEVX_SET64(flow_table_context, ft_ctx, + sw_owner_icm_root_0, ft_attr->icm_addr_rx); + DEVX_SET64(flow_table_context, ft_ctx, + sw_owner_icm_root_1, ft_attr->icm_addr_tx); + } else { + assert(false); + } } return mlx5dv_devx_obj_create(ctx, in, sizeof(in), out, sizeof(out)); } +static struct mlx5dv_devx_obj * +dr_devx_create_flow_group(struct ibv_context *ctx, + struct dr_devx_flow_group_attr *fg_attr) +{ + uint32_t out[DEVX_ST_SZ_DW(create_flow_group_out)] = {}; + uint32_t inlen = DEVX_ST_SZ_BYTES(create_flow_group_in); + struct mlx5dv_devx_obj *obj; + uint32_t *in; + + in = calloc(1, inlen); + if (!in) { + errno = ENOMEM; + return NULL; + } + + DEVX_SET(create_flow_group_in, in, opcode, MLX5_CMD_OP_CREATE_FLOW_GROUP); + DEVX_SET(create_flow_group_in, in, table_type, fg_attr->table_type); + DEVX_SET(create_flow_group_in, in, table_id, fg_attr->table_id); + + obj = mlx5dv_devx_obj_create(ctx, in, inlen, out, sizeof(out)); + free(in); + + return obj; +} + +static struct mlx5dv_devx_obj * +dr_devx_set_fte(struct ibv_context *ctx, + struct dr_devx_flow_fte_attr *fte_attr) +{ + uint32_t out[DEVX_ST_SZ_DW(set_fte_out)] = {}; + struct mlx5dv_devx_obj *obj; + uint32_t dest_entry_size; + void *in_flow_context; + uint32_t list_size; + uint8_t *in_dests; + uint32_t inlen; + uint32_t *in; + uint32_t i; + + dest_entry_size = DEVX_ST_SZ_BYTES(dest_format); + inlen = DEVX_ST_SZ_BYTES(set_fte_in) + fte_attr->dest_size * dest_entry_size; + in = calloc(1, inlen); + if (!in) { + errno = ENOMEM; + return NULL; + } + + DEVX_SET(set_fte_in, in, opcode, MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY); + DEVX_SET(set_fte_in, in, table_type, fte_attr->table_type); + DEVX_SET(set_fte_in, in, table_id, fte_attr->table_id); + + in_flow_context = DEVX_ADDR_OF(set_fte_in, in, flow_context); + DEVX_SET(flow_context, in_flow_context, group_id, fte_attr->group_id); + DEVX_SET(flow_context, in_flow_context, flow_tag, fte_attr->flow_tag); + DEVX_SET(flow_context, in_flow_context, action, fte_attr->action); + + in_dests = DEVX_ADDR_OF(flow_context, in_flow_context, destination); + if (fte_attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) { + list_size = 0; + + for (i = 0; i < fte_attr->dest_size; i++) { + uint32_t id; + uint32_t type = fte_attr->dest_arr[i].type; + + if (type == MLX5_FLOW_DEST_TYPE_COUNTER) + continue; + + switch (type) { + case MLX5_FLOW_DEST_TYPE_VPORT: + id = fte_attr->dest_arr[i].vport_num; + break; + case MLX5_FLOW_DEST_TYPE_TIR: + id = fte_attr->dest_arr[i].tir_num; + break; + default: + errno = EOPNOTSUPP; + goto err_out; + } + + DEVX_SET(dest_format, in_dests, destination_type, type); + DEVX_SET(dest_format, in_dests, destination_id, id); + in_dests += dest_entry_size; + list_size++; + } + + DEVX_SET(flow_context, in_flow_context, destination_list_size, list_size); + } + + if (fte_attr->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { + list_size = 0; + + for (i = 0; i < fte_attr->dest_size; i++) { + if (fte_attr->dest_arr[i].type != MLX5_FLOW_DEST_TYPE_COUNTER) + continue; + + DEVX_SET(flow_counter_list, in_dests, flow_counter_id, + fte_attr->dest_arr[i].counter_id); + in_dests += dest_entry_size; + list_size++; + } + + DEVX_SET(flow_context, in_flow_context, flow_counter_list_size, list_size); + } + + obj = mlx5dv_devx_obj_create(ctx, in, inlen, out, sizeof(out)); + + free(in); + return obj; + +err_out: + free(in); + return NULL; +} + +struct dr_devx_tbl * +dr_devx_create_always_hit_ft(struct ibv_context *ctx, + struct dr_devx_flow_table_attr *ft_attr, + struct dr_devx_flow_group_attr *fg_attr, + struct dr_devx_flow_fte_attr *fte_attr) +{ + struct mlx5dv_devx_obj *fte_dvo; + struct mlx5dv_devx_obj *fg_dvo; + struct mlx5dv_devx_obj *ft_dvo; + struct dr_devx_tbl *tbl; + + tbl = calloc(1, sizeof(*tbl)); + if (!tbl) { + errno = ENOMEM; + return NULL; + } + + ft_dvo = dr_devx_create_flow_table(ctx, ft_attr); + if (!ft_dvo) + goto free_tbl; + + fg_attr->table_id = ft_dvo->object_id; + fg_attr->table_type = ft_attr->type; + fg_dvo = dr_devx_create_flow_group(ctx, fg_attr); + if (!fg_dvo) + goto free_ft_dvo; + + fte_attr->table_id = ft_dvo->object_id; + fte_attr->table_type = ft_attr->type; + fte_attr->group_id = fg_dvo->object_id; + fte_dvo = dr_devx_set_fte(ctx, fte_attr); + if (!fte_dvo) + goto free_fg_dvo; + + tbl->type = ft_attr->type; + tbl->level = ft_attr->level; + tbl->ft_dvo = ft_dvo; + tbl->fg_dvo = fg_dvo; + tbl->fte_dvo = fte_dvo; + + return tbl; + +free_fg_dvo: + mlx5dv_devx_obj_destroy(fg_dvo); +free_ft_dvo: + mlx5dv_devx_obj_destroy(ft_dvo); +free_tbl: + free(tbl); + + return NULL; +} + +void dr_devx_destroy_always_hit_ft(struct dr_devx_tbl *devx_tbl) +{ + mlx5dv_devx_obj_destroy(devx_tbl->fte_dvo); + mlx5dv_devx_obj_destroy(devx_tbl->fg_dvo); + mlx5dv_devx_obj_destroy(devx_tbl->ft_dvo); + free(devx_tbl); +} + struct mlx5dv_devx_obj *dr_devx_create_reformat_ctx(struct ibv_context *ctx, enum reformat_type rt, size_t reformat_size, diff --git a/providers/mlx5/dr_table.c b/providers/mlx5/dr_table.c index 3f3a0658e..0e135b76a 100644 --- a/providers/mlx5/dr_table.c +++ b/providers/mlx5/dr_table.c @@ -151,20 +151,19 @@ static int dr_table_init(struct mlx5dv_dr_table *tbl) static int dr_table_create_devx_tbl(struct mlx5dv_dr_table *tbl) { - uint64_t icm_addr_rx = 0; - uint64_t icm_addr_tx = 0; + struct dr_devx_flow_table_attr ft_attr = {}; + + ft_attr.type = tbl->table_type; + ft_attr.level = tbl->dmn->info.caps.max_ft_level - 1; + ft_attr.sw_owner = true; if (tbl->rx.s_anchor) - icm_addr_rx = tbl->rx.s_anchor->chunk->icm_addr; + ft_attr.icm_addr_rx = tbl->rx.s_anchor->chunk->icm_addr; if (tbl->tx.s_anchor) - icm_addr_tx = tbl->tx.s_anchor->chunk->icm_addr; + ft_attr.icm_addr_tx = tbl->tx.s_anchor->chunk->icm_addr; - tbl->devx_obj = dr_devx_create_flow_table(tbl->dmn->ctx, - tbl->table_type, - icm_addr_rx, - icm_addr_tx, - tbl->dmn->info.caps.max_ft_level - 1); + tbl->devx_obj = dr_devx_create_flow_table(tbl->dmn->ctx, &ft_attr); if (!tbl->devx_obj) return errno; diff --git a/providers/mlx5/mlx5.h b/providers/mlx5/mlx5.h index af1ff073e..ee1ae51dc 100644 --- a/providers/mlx5/mlx5.h +++ b/providers/mlx5/mlx5.h @@ -668,6 +668,8 @@ enum mlx5_devx_obj_type { MLX5_DEVX_QP = 4, MLX5_DEVX_PKT_REFORMAT_CTX = 5, MLX5_DEVX_TIR = 6, + MLX5_DEVX_FLOW_GROUP = 7, + MLX5_DEVX_FLOW_TABLE_ENTRY = 8, }; struct mlx5dv_devx_obj { diff --git a/providers/mlx5/mlx5_ifc.h b/providers/mlx5/mlx5_ifc.h index e434d2cde..a986134dc 100644 --- a/providers/mlx5/mlx5_ifc.h +++ b/providers/mlx5/mlx5_ifc.h @@ -55,6 +55,8 @@ enum { MLX5_CMD_OP_MODIFY_TIS = 0x913, MLX5_CMD_OP_QUERY_TIS = 0x915, MLX5_CMD_OP_CREATE_FLOW_TABLE = 0x930, + MLX5_CMD_OP_CREATE_FLOW_GROUP = 0x933, + MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY = 0x936, MLX5_CMD_OP_CREATE_FLOW_COUNTER = 0x939, MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT = 0x93d, MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT = 0x93e, @@ -105,7 +107,7 @@ struct mlx5_ifc_flow_table_context_bits { u8 reformat_en[0x1]; u8 decap_en[0x1]; u8 sw_owner[0x1]; - u8 reserved_at_3[0x1]; + u8 termination_table[0x1]; u8 table_miss_action[0x4]; u8 level[0x8]; u8 reserved_at_10[0x8]; @@ -2512,6 +2514,114 @@ struct mlx5_ifc_dr_action_hw_copy_bits { u8 reserved_at_38[0x8]; }; +struct mlx5_ifc_create_flow_group_in_bits { + u8 opcode[0x10]; + u8 reserved_at_10[0x10]; + + u8 reserved_at_20[0x60]; + + u8 table_type[0x8]; + u8 reserved_at_88[0x18]; + + u8 reserved_at_a0[0x8]; + u8 table_id[0x18]; + + u8 reserved_at_c0[0x1f40]; +}; + +struct mlx5_ifc_create_flow_group_out_bits { + u8 status[0x8]; + u8 reserved_at_8[0x18]; + + u8 syndrome[0x20]; + + u8 reserved_at_40[0x8]; + u8 group_id[0x18]; + + u8 reserved_at_60[0x20]; +}; + +struct mlx5_ifc_dest_format_bits { + u8 destination_type[0x8]; + u8 destination_id[0x18]; + + u8 reserved_at_20[0x20]; +}; + +struct mlx5_ifc_flow_counter_list_bits { + u8 flow_counter_id[0x20]; + + u8 reserved_at_20[0x20]; +}; + +union mlx5_ifc_dest_format_flow_counter_list_auto_bits { + struct mlx5_ifc_dest_format_bits dest_format; + struct mlx5_ifc_flow_counter_list_bits flow_counter_list; + u8 reserved_at_0[0x40]; +}; + +struct mlx5_ifc_flow_context_bits { + u8 reserved_at_00[0x20]; + + u8 group_id[0x20]; + + u8 reserved_at_40[0x8]; + u8 flow_tag[0x18]; + + u8 reserved_at_60[0x10]; + u8 action[0x10]; + + u8 reserved_at_80[0x8]; + u8 destination_list_size[0x18]; + + u8 reserved_at_a0[0x8]; + u8 flow_counter_list_size[0x18]; + + u8 reserved_at_c0[0x1740]; + + union mlx5_ifc_dest_format_flow_counter_list_auto_bits destination[0]; +}; + +struct mlx5_ifc_set_fte_in_bits { + u8 opcode[0x10]; + u8 reserved_at_10[0x10]; + + u8 reserved_at_20[0x60]; + + u8 table_type[0x8]; + u8 reserved_at_88[0x18]; + + u8 reserved_at_a0[0x8]; + u8 table_id[0x18]; + + u8 reserved_at_c0[0x40]; + u8 flow_index[0x20]; + + u8 reserved_at_120[0xe0]; + struct mlx5_ifc_flow_context_bits flow_context; +}; + +struct mlx5_ifc_set_fte_out_bits { + u8 status[0x8]; + u8 reserved_at_8[0x18]; + + u8 syndrome[0x20]; + + u8 reserved_at_40[0x40]; +}; + +enum dr_devx_flow_dest_type { + MLX5_FLOW_DEST_TYPE_VPORT = 0x0, + MLX5_FLOW_DEST_TYPE_TIR = 0x2, + + MLX5_FLOW_DEST_TYPE_COUNTER = 0x100, +}; + +enum { + MLX5_FLOW_CONTEXT_ACTION_FWD_DEST = 0x4, + MLX5_FLOW_CONTEXT_ACTION_COUNT = 0x8, +}; + enum { MLX5_QPC_PAGE_OFFSET_QUANTA = 64, }; diff --git a/providers/mlx5/mlx5dv_dr.h b/providers/mlx5/mlx5dv_dr.h index 725f1423f..3064e9713 100644 --- a/providers/mlx5/mlx5dv_dr.h +++ b/providers/mlx5/mlx5dv_dr.h @@ -634,6 +634,47 @@ struct dr_devx_caps { struct dr_devx_roce_cap roce_caps; }; +struct dr_devx_flow_table_attr { + uint8_t type; + uint8_t level; + bool sw_owner; + bool term_tbl; + uint64_t icm_addr_rx; + uint64_t icm_addr_tx; +}; + +struct dr_devx_flow_group_attr { + uint32_t table_id; + uint32_t table_type; +}; + +struct dr_devx_flow_dest_info { + enum dr_devx_flow_dest_type type; + union { + uint32_t vport_num; + uint32_t tir_num; + uint32_t counter_id; + }; +}; + +struct dr_devx_flow_fte_attr { + uint32_t table_id; + uint32_t table_type; + uint32_t group_id; + uint32_t flow_tag; + uint32_t action; + uint32_t dest_size; + struct dr_devx_flow_dest_info *dest_arr; +}; + +struct dr_devx_tbl { + uint8_t type; + uint8_t level; + struct mlx5dv_devx_obj *ft_dvo; + struct mlx5dv_devx_obj *fg_dvo; + struct mlx5dv_devx_obj *fte_dvo; +}; + struct dr_domain_rx_tx { uint64_t drop_icm_addr; uint64_t default_icm_addr; @@ -896,11 +937,16 @@ int dr_devx_query_gvmi(struct ibv_context *ctx, int dr_devx_query_esw_caps(struct ibv_context *ctx, struct dr_esw_caps *caps); int dr_devx_sync_steering(struct ibv_context *ctx); -struct mlx5dv_devx_obj *dr_devx_create_flow_table(struct ibv_context *ctx, - uint32_t table_type, - uint64_t icm_addr_rx, - uint64_t icm_addr_tx, - u8 level); +struct mlx5dv_devx_obj * +dr_devx_create_flow_table(struct ibv_context *ctx, + struct dr_devx_flow_table_attr *table_attr); + +struct dr_devx_tbl * +dr_devx_create_always_hit_ft(struct ibv_context *ctx, + struct dr_devx_flow_table_attr *ft_attr, + struct dr_devx_flow_group_attr *fg_attr, + struct dr_devx_flow_fte_attr *fte_attr); +void dr_devx_destroy_always_hit_ft(struct dr_devx_tbl *devx_tbl); struct mlx5dv_devx_obj *dr_devx_create_reformat_ctx(struct ibv_context *ctx, enum reformat_type rt, size_t reformat_size, diff --git a/providers/mlx5/verbs.c b/providers/mlx5/verbs.c index a996a99a3..fecc356d7 100644 --- a/providers/mlx5/verbs.c +++ b/providers/mlx5/verbs.c @@ -4754,6 +4754,14 @@ static void set_devx_obj_info(const void *in, const void *out, obj->type = MLX5_DEVX_FLOW_TABLE; obj->object_id = DEVX_GET(create_flow_table_out, out, table_id); break; + case MLX5_CMD_OP_CREATE_FLOW_GROUP: + obj->type = MLX5_DEVX_FLOW_GROUP; + obj->object_id = DEVX_GET(create_flow_group_out, out, group_id); + break; + case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY: + obj->type = MLX5_DEVX_FLOW_TABLE_ENTRY; + obj->object_id = DEVX_GET(set_fte_in, in, flow_index); + break; case MLX5_CMD_OP_CREATE_FLOW_COUNTER: obj->type = MLX5_DEVX_FLOW_COUNTER; obj->object_id = DEVX_GET(alloc_flow_counter_out, out, flow_counter_id); From b2bffc7761de48550fa5ba1fb28c3773fe2aee10 Mon Sep 17 00:00:00 2001 From: Shun Hao Date: Tue, 1 Sep 2020 11:50:48 +0300 Subject: [PATCH 2/2] mlx5: DR, Add support for flow sampler action The sampler action is used for sampling some ratio of the packets without disrupting the traffic. When a packet enters the sampler object it will be duplicated and sent to the sampling flow table as well as the default flow table. On the sampler table we can execute other dr action on the sampled packet. It is possible to set a single modify action which can be done before forwarding packet to the default flow table. Currently it is required to provide a restore action for FDB sampler, in order to restore vport alias for packets steered to default flow table. More details can be found in mlx5dv_dr_flow man page. Signed-off-by: Shun Hao Reviewed-by: Alex Vesker Signed-off-by: Yishai Hadas --- debian/ibverbs-providers.symbols | 2 + providers/mlx5/CMakeLists.txt | 2 +- providers/mlx5/dr_action.c | 427 +++++++++++++++++++++++++ providers/mlx5/dr_dbg.c | 14 + providers/mlx5/dr_devx.c | 58 ++++ providers/mlx5/libmlx5.map | 5 + providers/mlx5/man/CMakeLists.txt | 1 + providers/mlx5/man/mlx5dv_dr_flow.3.md | 11 + providers/mlx5/mlx5.h | 1 + providers/mlx5/mlx5_ifc.h | 31 ++ providers/mlx5/mlx5dv.h | 11 + providers/mlx5/mlx5dv_dr.h | 43 +++ providers/mlx5/verbs.c | 2 + 13 files changed, 607 insertions(+), 1 deletion(-) diff --git a/debian/ibverbs-providers.symbols b/debian/ibverbs-providers.symbols index 8371603b2..6268f513c 100644 --- a/debian/ibverbs-providers.symbols +++ b/debian/ibverbs-providers.symbols @@ -23,6 +23,7 @@ libmlx5.so.1 ibverbs-providers #MINVER# MLX5_1.13@MLX5_1.13 29 MLX5_1.14@MLX5_1.14 30 MLX5_1.15@MLX5_1.15 31 + MLX5_1.16@MLX5_1.16 32 mlx5dv_init_obj@MLX5_1.0 13 mlx5dv_init_obj@MLX5_1.2 15 mlx5dv_query_device@MLX5_1.0 13 @@ -105,6 +106,7 @@ libmlx5.so.1 ibverbs-providers #MINVER# mlx5dv_modify_qp_lag_port@MLX5_1.14 30 mlx5dv_query_qp_lag_port@MLX5_1.14 30 mlx5dv_dr_action_create_dest_devx_tir@MLX5_1.15 31 + mlx5dv_dr_action_create_flow_sampler@MLX5_1.16 32 libefa.so.1 ibverbs-providers #MINVER# * Build-Depends-Package: libibverbs-dev EFA_1.0@EFA_1.0 24 diff --git a/providers/mlx5/CMakeLists.txt b/providers/mlx5/CMakeLists.txt index 460a64d84..3c68c1db5 100644 --- a/providers/mlx5/CMakeLists.txt +++ b/providers/mlx5/CMakeLists.txt @@ -11,7 +11,7 @@ if (MLX5_MW_DEBUG) endif() rdma_shared_provider(mlx5 libmlx5.map - 1 1.15.${PACKAGE_VERSION} + 1 1.16.${PACKAGE_VERSION} buf.c cq.c dbrec.c diff --git a/providers/mlx5/dr_action.c b/providers/mlx5/dr_action.c index 7bdf3961b..a5771695f 100644 --- a/providers/mlx5/dr_action.c +++ b/providers/mlx5/dr_action.c @@ -64,6 +64,7 @@ static const enum dr_action_valid_state next_action_state[DR_ACTION_DOMAIN_MAX] [DR_ACTION_TYP_TAG] = DR_ACTION_STATE_NON_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_NON_TERM, [DR_ACTION_TYP_METER] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_TNL_L2_TO_L2] = DR_ACTION_STATE_REFORMAT, [DR_ACTION_TYP_TNL_L3_TO_L2] = DR_ACTION_STATE_REFORMAT, [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, @@ -75,6 +76,7 @@ static const enum dr_action_valid_state next_action_state[DR_ACTION_DOMAIN_MAX] [DR_ACTION_TYP_TAG] = DR_ACTION_STATE_REFORMAT, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_REFORMAT, [DR_ACTION_TYP_METER] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, }, [DR_ACTION_STATE_MODIFY_HDR] = { @@ -83,6 +85,7 @@ static const enum dr_action_valid_state next_action_state[DR_ACTION_DOMAIN_MAX] [DR_ACTION_TYP_TAG] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_METER] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, }, [DR_ACTION_STATE_NON_TERM] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, @@ -91,6 +94,7 @@ static const enum dr_action_valid_state next_action_state[DR_ACTION_DOMAIN_MAX] [DR_ACTION_TYP_TAG] = DR_ACTION_STATE_NON_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_NON_TERM, [DR_ACTION_TYP_METER] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_TNL_L2_TO_L2] = DR_ACTION_STATE_REFORMAT, [DR_ACTION_TYP_TNL_L3_TO_L2] = DR_ACTION_STATE_REFORMAT, [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, @@ -143,6 +147,7 @@ static const enum dr_action_valid_state next_action_state[DR_ACTION_DOMAIN_MAX] [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_NON_TERM, [DR_ACTION_TYP_METER] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_TNL_L2_TO_L2] = DR_ACTION_STATE_REFORMAT, [DR_ACTION_TYP_TNL_L3_TO_L2] = DR_ACTION_STATE_REFORMAT, [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, @@ -153,6 +158,7 @@ static const enum dr_action_valid_state next_action_state[DR_ACTION_DOMAIN_MAX] [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_REFORMAT, [DR_ACTION_TYP_METER] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, }, @@ -160,6 +166,7 @@ static const enum dr_action_valid_state next_action_state[DR_ACTION_DOMAIN_MAX] [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_METER] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, }, [DR_ACTION_STATE_NON_TERM] = { @@ -167,6 +174,7 @@ static const enum dr_action_valid_state next_action_state[DR_ACTION_DOMAIN_MAX] [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_NON_TERM, [DR_ACTION_TYP_METER] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_TNL_L2_TO_L2] = DR_ACTION_STATE_REFORMAT, [DR_ACTION_TYP_TNL_L3_TO_L2] = DR_ACTION_STATE_REFORMAT, [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, @@ -184,6 +192,7 @@ static const enum dr_action_valid_state next_action_state[DR_ACTION_DOMAIN_MAX] [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_NON_TERM, [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_METER] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_REFORMAT, [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_REFORMAT, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, @@ -193,12 +202,14 @@ static const enum dr_action_valid_state next_action_state[DR_ACTION_DOMAIN_MAX] [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_REFORMAT, [DR_ACTION_TYP_METER] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, }, [DR_ACTION_STATE_MODIFY_HDR] = { [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_METER] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_REFORMAT, [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_REFORMAT, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, @@ -209,6 +220,7 @@ static const enum dr_action_valid_state next_action_state[DR_ACTION_DOMAIN_MAX] [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_NON_TERM, [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_METER] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_REFORMAT, [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_REFORMAT, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, @@ -423,6 +435,25 @@ int dr_actions_build_ste_arr(struct mlx5dv_dr_matcher *matcher, action->meter.rx_icm_addr : action->meter.tx_icm_addr; break; + case DR_ACTION_TYP_SAMPLER: + if (action->sampler.dmn != dmn) { + dr_dbg(dmn, "Sampler belongs to a different domain\n"); + goto out_invalid_arg; + } + if (action->sampler.sampler_default->next_ft->level <= + matcher->tbl->level) { + dr_dbg(dmn, "Sampler next table level should he higher than source table\n"); + goto out_invalid_arg; + } + + if (rx_rule) { + attr.final_icm_addr = action->sampler.sampler_default->rx_icm_addr; + } else { + attr.final_icm_addr = (action->sampler.sampler_restore) ? + action->sampler.sampler_restore->tx_icm_addr : + action->sampler.sampler_default->tx_icm_addr; + } + break; case DR_ACTION_TYP_VPORT: if (action->vport.dmn != dmn) { dr_dbg(dmn, "Destination vport belongs to a different domain\n"); @@ -1512,6 +1543,393 @@ struct mlx5dv_dr_action return action; } +static struct dr_devx_tbl_with_refs * +dr_action_create_sampler_term_tbl(struct mlx5dv_dr_domain *dmn, + struct mlx5dv_dr_flow_sampler_attr *attr) +{ + struct dr_devx_flow_table_attr ft_attr = {}; + struct dr_devx_flow_group_attr fg_attr = {}; + struct dr_devx_flow_fte_attr fte_attr = {}; + struct dr_devx_flow_dest_info *dest_info; + struct dr_devx_tbl_with_refs *term_tbl; + struct mlx5dv_dr_action **ref_actions; + uint32_t dest_index = 0; + uint32_t ref_index = 0; + uint32_t tbl_type; + uint32_t i; + + tbl_type = attr->default_next_table->table_type; + + dest_info = calloc(attr->num_sample_actions, + sizeof(struct dr_devx_flow_dest_info)); + if (!dest_info) { + errno = ENOMEM; + return NULL; + } + + term_tbl = calloc(1, sizeof(struct dr_devx_tbl_with_refs)); + if (!term_tbl) { + errno = ENOMEM; + goto free_dest_info; + } + + ref_actions = calloc(attr->num_sample_actions, + sizeof(struct mlx5dv_dr_action *)); + if (!ref_actions) { + errno = ENOMEM; + goto free_term_tbl; + } + + for (i = 0; i < attr->num_sample_actions; i++) { + enum dr_action_type action_type = + attr->sample_actions[i]->action_type; + + atomic_fetch_add(&attr->sample_actions[i]->refcount, 1); + ref_actions[ref_index++] = attr->sample_actions[i]; + + switch (action_type) { + case DR_ACTION_TYP_MISS: + case DR_ACTION_TYP_VPORT: + if (tbl_type != FS_FT_FDB) { + errno = EOPNOTSUPP; + goto free_ref_actions; + } + + fte_attr.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + dest_info[dest_index].type = MLX5_FLOW_DEST_TYPE_VPORT; + + if (action_type == DR_ACTION_TYP_MISS) + dest_info[dest_index].vport_num = 0; + else + dest_info[dest_index].vport_num = + attr->sample_actions[i]->vport.num; + + dest_index++; + break; + case DR_ACTION_TYP_QP: + if (tbl_type != FS_FT_NIC_RX) { + errno = EOPNOTSUPP; + goto free_ref_actions; + } + + fte_attr.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + dest_info[dest_index].type = MLX5_FLOW_DEST_TYPE_TIR; + + if (attr->sample_actions[i]->dest_qp.is_qp) { + struct mlx5_qp *mlx5_qp = + to_mqp(attr->sample_actions[i]->dest_qp.qp); + dest_info[dest_index].tir_num = mlx5_qp->tirn; + } else { + dest_info[dest_index].tir_num = + attr->sample_actions[i]->dest_qp.devx_tir->object_id; + } + dest_index++; + break; + case DR_ACTION_TYP_CTR: + if (tbl_type != FS_FT_NIC_RX) { + errno = EOPNOTSUPP; + goto free_ref_actions; + } + + fte_attr.action |= MLX5_FLOW_CONTEXT_ACTION_COUNT; + dest_info[dest_index].type = MLX5_FLOW_DEST_TYPE_COUNTER; + dest_info[dest_index].counter_id = + attr->sample_actions[i]->ctr.devx_obj->object_id + + attr->sample_actions[i]->ctr.offset; + dest_index++; + break; + case DR_ACTION_TYP_TAG: + if (tbl_type != FS_FT_NIC_RX) { + errno = EOPNOTSUPP; + goto free_ref_actions; + } + + fte_attr.flow_tag = attr->sample_actions[i]->flow_tag; + break; + default: + errno = EOPNOTSUPP; + goto free_ref_actions; + } + } + + ft_attr.type = tbl_type; + ft_attr.level = dmn->info.caps.max_ft_level - 1; + ft_attr.term_tbl = true; + fte_attr.dest_size = dest_index; + fte_attr.dest_arr = dest_info; + term_tbl->devx_tbl = dr_devx_create_always_hit_ft(dmn->ctx, &ft_attr, + &fg_attr, &fte_attr); + if (!term_tbl->devx_tbl) + goto free_ref_actions; + + term_tbl->ref_actions = ref_actions; + term_tbl->ref_actions_num = attr->num_sample_actions; + + free(dest_info); + return term_tbl; + +free_ref_actions: + for (i = 0; i < ref_index; i++) + atomic_fetch_sub(&ref_actions[i]->refcount, 1); + free(ref_actions); +free_term_tbl: + free(term_tbl); +free_dest_info: + free(dest_info); + + return NULL; +} + +static void +dr_action_destroy_sampler_term_tbl(struct dr_devx_tbl_with_refs *term_tbl) +{ + uint32_t i; + + dr_devx_destroy_always_hit_ft(term_tbl->devx_tbl); + + for (i = 0; i < term_tbl->ref_actions_num; i++) + atomic_fetch_sub(&term_tbl->ref_actions[i]->refcount, 1); + free(term_tbl->ref_actions); + free(term_tbl); +} + +static struct dr_flow_sampler * +dr_action_create_sampler(struct mlx5dv_dr_domain *dmn, + struct mlx5dv_dr_flow_sampler_attr *attr, + struct dr_devx_tbl_with_refs *term_tbl, + struct dr_flow_sampler_restore_tbl *restore) +{ + struct dr_devx_flow_sampler_attr sampler_attr = {}; + struct dr_flow_sampler *sampler; + uint64_t icm_rx, icm_tx; + int ret; + + sampler = calloc(1, sizeof(struct dr_flow_sampler)); + if (!sampler) { + errno = ENOMEM; + return NULL; + } + + sampler->next_ft = restore ? restore->tbl : attr->default_next_table; + atomic_fetch_add(&sampler->next_ft->refcount, 1); + + /* Sampler HW level equals to term_tbl HW level, need to set ignore level */ + sampler_attr.ignore_flow_level = true; + sampler_attr.sample_ratio = attr->sample_ratio; + sampler_attr.table_type = term_tbl->devx_tbl->type; + sampler_attr.level = term_tbl->devx_tbl->level; + sampler_attr.sample_table_id = term_tbl->devx_tbl->ft_dvo->object_id; + sampler_attr.default_next_table_id = sampler->next_ft->devx_obj->object_id; + + sampler->devx_obj = dr_devx_create_flow_sampler(dmn->ctx, &sampler_attr); + if (!sampler->devx_obj) + goto dec_next_ft_ref; + + ret = dr_devx_query_flow_sampler(sampler->devx_obj, &icm_rx, &icm_tx); + if (ret) + goto destroy_sampler_dvo; + + sampler->rx_icm_addr = icm_rx; + sampler->tx_icm_addr = icm_tx; + + return sampler; + +destroy_sampler_dvo: + mlx5dv_devx_obj_destroy(sampler->devx_obj); +dec_next_ft_ref: + atomic_fetch_sub(&sampler->next_ft->refcount, 1); + + free(sampler); + + return NULL; +} + +static void dr_action_destroy_sampler(struct dr_flow_sampler *sampler) +{ + mlx5dv_devx_obj_destroy(sampler->devx_obj); + atomic_fetch_sub(&sampler->next_ft->refcount, 1); + free(sampler); +} + +static struct dr_flow_sampler_restore_tbl * +dr_action_create_sampler_restore_tbl(struct mlx5dv_dr_domain *dmn, + struct mlx5dv_dr_flow_sampler_attr *attr) +{ + struct mlx5dv_flow_match_parameters *mask; + struct dr_flow_sampler_restore_tbl *restore; + uint32_t action_field; + uint32_t action_type; + uint32_t mask_size; + + action_type = DEVX_GET(set_action_in, &(attr->action), action_type); + action_field = DEVX_GET(set_action_in, &(attr->action), field); + + /* Currently only support restore of setting Reg_C0 */ + if (action_type != MLX5_ACTION_TYPE_SET || + action_field != MLX5_ACTION_IN_FIELD_OUT_METADATA_REGC_0) { + errno = EOPNOTSUPP; + return NULL; + } + + mask_size = sizeof(struct mlx5dv_flow_match_parameters) + + sizeof(struct dr_match_param); + mask = calloc(1, mask_size); + if (!mask) { + errno = ENOMEM; + return NULL; + } + mask->match_sz = sizeof(struct dr_match_param); + + restore = calloc(1, sizeof(struct dr_flow_sampler_restore_tbl)); + if (!restore) { + errno = ENOMEM; + goto free_mask; + } + + restore->tbl = mlx5dv_dr_table_create(dmn, attr->default_next_table->level - 1); + if (!restore->tbl) + goto free_restore; + + restore->matcher = mlx5dv_dr_matcher_create(restore->tbl, 0, 0, mask); + if (!restore->matcher) + goto destroy_restore_tbl; + + restore->num_of_actions = 2; + restore->actions = calloc(restore->num_of_actions, + sizeof(struct mlx5dv_dr_action *)); + if (!restore->actions) { + errno = ENOMEM; + goto destroy_restore_matcher; + } + + restore->actions[0] = + mlx5dv_dr_action_create_modify_header(dmn, 0, + DR_MODIFY_ACTION_SIZE, + &(attr->action)); + if (!restore->actions[0]) + goto free_action_list; + + restore->actions[1] = + mlx5dv_dr_action_create_dest_table(attr->default_next_table); + if (!restore->actions[1]) + goto destroy_modify_hdr_action; + + restore->rule = mlx5dv_dr_rule_create(restore->matcher, mask, + restore->num_of_actions, + restore->actions); + if (!restore->rule) + goto destroy_dest_action; + + free(mask); + return restore; + +destroy_dest_action: + mlx5dv_dr_action_destroy(restore->actions[1]); +destroy_modify_hdr_action: + mlx5dv_dr_action_destroy(restore->actions[0]); +free_action_list: + free(restore->actions); +destroy_restore_matcher: + mlx5dv_dr_matcher_destroy(restore->matcher); +destroy_restore_tbl: + mlx5dv_dr_table_destroy(restore->tbl); +free_restore: + free(restore); +free_mask: + free(mask); + + return NULL; +} + +static void dr_action_destroy_sampler_restore_tbl(struct dr_flow_sampler_restore_tbl *restore) +{ + uint32_t i; + + mlx5dv_dr_rule_destroy(restore->rule); + for (i = 0; i < restore->num_of_actions; i++) + mlx5dv_dr_action_destroy(restore->actions[i]); + free(restore->actions); + + mlx5dv_dr_matcher_destroy(restore->matcher); + mlx5dv_dr_table_destroy(restore->tbl); + free(restore); +} + +struct mlx5dv_dr_action * +mlx5dv_dr_action_create_flow_sampler(struct mlx5dv_dr_flow_sampler_attr *attr) +{ + struct mlx5dv_dr_action *action; + struct mlx5dv_dr_domain *dmn; + bool restore = false; + + dmn = attr->default_next_table->dmn; + if (!dmn || + !attr->default_next_table || attr->sample_ratio == 0 || + !attr->sample_actions || attr->num_sample_actions == 0) { + errno = EINVAL; + return NULL; + } + + if (dmn->type != MLX5DV_DR_DOMAIN_TYPE_NIC_RX && + dmn->type != MLX5DV_DR_DOMAIN_TYPE_FDB) { + errno = EOPNOTSUPP; + return NULL; + } + + if (dmn->type == MLX5DV_DR_DOMAIN_TYPE_FDB) + restore = true; + + atomic_fetch_add(&dmn->refcount, 1); + + action = dr_action_create_generic(DR_ACTION_TYP_SAMPLER); + if (!action) + goto dec_ref; + + action->sampler.dmn = dmn; + + action->sampler.term_tbl = dr_action_create_sampler_term_tbl(dmn, attr); + if (!action->sampler.term_tbl) + goto free_action; + + action->sampler.sampler_default = dr_action_create_sampler(dmn, attr, + action->sampler.term_tbl, + NULL); + if (!action->sampler.sampler_default) + goto destroy_term_tbl; + + if (restore) { + struct dr_flow_sampler *sampler_restore; + + action->sampler.restore_tbl = dr_action_create_sampler_restore_tbl(dmn, attr); + if (!action->sampler.restore_tbl) + goto destroy_sampler_default; + + sampler_restore = dr_action_create_sampler(dmn, attr, + action->sampler.term_tbl, + action->sampler.restore_tbl); + if (!sampler_restore) + goto destroy_restore; + + action->sampler.sampler_restore = sampler_restore; + } + + return action; + +destroy_restore: + if (action->sampler.restore_tbl) + dr_action_destroy_sampler_restore_tbl(action->sampler.restore_tbl); +destroy_sampler_default: + dr_action_destroy_sampler(action->sampler.sampler_default); +destroy_term_tbl: + dr_action_destroy_sampler_term_tbl(action->sampler.term_tbl); +free_action: + free(action); +dec_ref: + atomic_fetch_sub(&dmn->refcount, 1); + + return NULL; +} + int mlx5dv_dr_action_destroy(struct mlx5dv_dr_action *action) { if (atomic_load(&action->refcount) > 1) @@ -1554,6 +1972,15 @@ int mlx5dv_dr_action_destroy(struct mlx5dv_dr_action *action) mlx5dv_devx_obj_destroy(action->meter.devx_obj); atomic_fetch_sub(&action->meter.next_ft->refcount, 1); break; + case DR_ACTION_TYP_SAMPLER: + if (action->sampler.sampler_restore) { + dr_action_destroy_sampler(action->sampler.sampler_restore); + dr_action_destroy_sampler_restore_tbl(action->sampler.restore_tbl); + } + dr_action_destroy_sampler(action->sampler.sampler_default); + dr_action_destroy_sampler_term_tbl(action->sampler.term_tbl); + atomic_fetch_sub(&action->sampler.dmn->refcount, 1); + break; default: break; } diff --git a/providers/mlx5/dr_dbg.c b/providers/mlx5/dr_dbg.c index 4b7463b42..9412d96ad 100644 --- a/providers/mlx5/dr_dbg.c +++ b/providers/mlx5/dr_dbg.c @@ -71,6 +71,7 @@ enum dr_dump_rec_type { DR_DUMP_REC_TYPE_ACTION_DECAP_L3 = 3410, DR_DUMP_REC_TYPE_ACTION_DEVX_TIR = 3411, DR_DUMP_REC_TYPE_ACTION_METER = 3414, + DR_DUMP_REC_TYPE_ACTION_SAMPLER = 3415, }; static uint64_t dr_dump_icm_to_idx(uint64_t icm_addr) @@ -165,6 +166,19 @@ static int dr_dump_rule_action_mem(FILE *f, const uint64_t rule_id, action->meter.rx_icm_addr, action->meter.tx_icm_addr); break; + case DR_ACTION_TYP_SAMPLER: + ret = fprintf(f, "%d,0x%" PRIx64 ",0x%" PRIx64 ",0x%" PRIx64 ",0x%x,0x%x,0x%" PRIx64 ",0x%" PRIx64 "\n", + DR_DUMP_REC_TYPE_ACTION_SAMPLER, + action_id, + rule_id, + (uint64_t)(uintptr_t)action->sampler.sampler_default->next_ft, + action->sampler.term_tbl->devx_tbl->ft_dvo->object_id, + action->sampler.sampler_default->devx_obj->object_id, + action->sampler.sampler_default->rx_icm_addr, + (action->sampler.sampler_restore) ? + action->sampler.sampler_restore->tx_icm_addr : + action->sampler.sampler_default->tx_icm_addr); + break; default: return 0; } diff --git a/providers/mlx5/dr_devx.c b/providers/mlx5/dr_devx.c index af372301f..94b7fe0e4 100644 --- a/providers/mlx5/dr_devx.c +++ b/providers/mlx5/dr_devx.c @@ -470,6 +470,64 @@ void dr_devx_destroy_always_hit_ft(struct dr_devx_tbl *devx_tbl) free(devx_tbl); } +struct mlx5dv_devx_obj * +dr_devx_create_flow_sampler(struct ibv_context *ctx, + struct dr_devx_flow_sampler_attr *sampler_attr) +{ + uint32_t out[DEVX_ST_SZ_DW(general_obj_out_cmd_hdr)] = {}; + uint32_t in[DEVX_ST_SZ_DW(create_flow_sampler_in)] = {}; + void *attr; + + attr = DEVX_ADDR_OF(create_flow_sampler_in, in, hdr); + DEVX_SET(general_obj_in_cmd_hdr, + attr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); + DEVX_SET(general_obj_in_cmd_hdr, + attr, obj_type, MLX5_OBJ_TYPE_FLOW_SAMPLER); + + attr = DEVX_ADDR_OF(create_flow_sampler_in, in, sampler); + DEVX_SET(flow_sampler, attr, table_type, sampler_attr->table_type); + DEVX_SET(flow_sampler, attr, level, sampler_attr->level); + DEVX_SET(flow_sampler, attr, sample_ratio, sampler_attr->sample_ratio); + DEVX_SET(flow_sampler, attr, ignore_flow_level, + sampler_attr->ignore_flow_level); + DEVX_SET(flow_sampler, attr, default_table_id, + sampler_attr->default_next_table_id); + DEVX_SET(flow_sampler, attr, sample_table_id, + sampler_attr->sample_table_id); + + return mlx5dv_devx_obj_create(ctx, in, sizeof(in), out, sizeof(out)); +} + +int dr_devx_query_flow_sampler(struct mlx5dv_devx_obj *obj, + uint64_t *rx_icm_addr, uint64_t *tx_icm_addr) +{ + uint32_t out[DEVX_ST_SZ_DW(query_flow_sampler_out)] = {}; + uint32_t in[DEVX_ST_SZ_DW(general_obj_in_cmd_hdr)] = {}; + void *attr; + int ret; + + DEVX_SET(general_obj_in_cmd_hdr, in, opcode, + MLX5_CMD_OP_QUERY_GENERAL_OBJECT); + DEVX_SET(general_obj_in_cmd_hdr, in, obj_type, + MLX5_OBJ_TYPE_FLOW_SAMPLER); + DEVX_SET(general_obj_in_cmd_hdr, in, obj_id, obj->object_id); + + ret = mlx5dv_devx_obj_query(obj, in, sizeof(in), out, sizeof(out)); + if (ret) { + dr_dbg_ctx(obj->context, "Failed to query flow sampler id %u\n", + obj->object_id); + return ret; + } + + attr = DEVX_ADDR_OF(query_flow_sampler_out, out, obj); + *rx_icm_addr = DEVX_GET64(flow_sampler, attr, + sw_steering_icm_address_rx); + *tx_icm_addr = DEVX_GET64(flow_sampler, attr, + sw_steering_icm_address_tx); + + return 0; +} + struct mlx5dv_devx_obj *dr_devx_create_reformat_ctx(struct ibv_context *ctx, enum reformat_type rt, size_t reformat_size, diff --git a/providers/mlx5/libmlx5.map b/providers/mlx5/libmlx5.map index a0e0b0eaf..702fcc3a9 100644 --- a/providers/mlx5/libmlx5.map +++ b/providers/mlx5/libmlx5.map @@ -146,3 +146,8 @@ MLX5_1.15 { global: mlx5dv_dr_action_create_dest_devx_tir; } MLX5_1.14; + +MLX5_1.16 { + global: + mlx5dv_dr_action_create_flow_sampler; +} MLX5_1.15; diff --git a/providers/mlx5/man/CMakeLists.txt b/providers/mlx5/man/CMakeLists.txt index 1d850cd17..666062481 100644 --- a/providers/mlx5/man/CMakeLists.txt +++ b/providers/mlx5/man/CMakeLists.txt @@ -62,6 +62,7 @@ rdma_alias_man_pages( mlx5dv_dr_flow.3 mlx5dv_dr_action_create_flow_counter.3 mlx5dv_dr_flow.3 mlx5dv_dr_action_create_drop.3 mlx5dv_dr_flow.3 mlx5dv_dr_action_create_default_miss.3 + mlx5dv_dr_flow.3 mlx5dv_dr_action_create_flow_sampler.3 mlx5dv_dr_flow.3 mlx5dv_dr_action_create_flow_meter.3 mlx5dv_dr_flow.3 mlx5dv_dr_action_create_modify_header.3 mlx5dv_dr_flow.3 mlx5dv_dr_action_create_packet_reformat.3 diff --git a/providers/mlx5/man/mlx5dv_dr_flow.3.md b/providers/mlx5/man/mlx5dv_dr_flow.3.md index 69878e2e2..dd3a3dbc5 100644 --- a/providers/mlx5/man/mlx5dv_dr_flow.3.md +++ b/providers/mlx5/man/mlx5dv_dr_flow.3.md @@ -34,6 +34,8 @@ mlx5dv_dr_action_create_flow_counter - Create devx flow counter actions mlx5dv_dr_action_create_flow_meter, mlx5dv_dr_action_modify_flow_meter - Create and modify meter action +mlx5dv_dr_action_create_flow_sampler - Create flow sampler action + mlx5dv_dr_action_destroy - Destroy actions # SYNOPSIS @@ -120,6 +122,9 @@ int mlx5dv_dr_action_modify_flow_meter(struct mlx5dv_dr_action *action, struct mlx5dv_dr_flow_meter_attr *attr, __be64 modify_field_select); +struct mlx5dv_dr_action * +mlx5dv_dr_action_create_flow_sampler(struct mlx5dv_dr_flow_sampler_attr *attr); + int mlx5dv_dr_action_destroy(struct mlx5dv_dr_action *action); ``` @@ -207,6 +212,12 @@ Action: Meter *mlx5dv_dr_action_create_flow_meter* creates a meter action based on the flow meter parameters. The paramertes are according to the device specification. *mlx5dv_dr_action_modify_flow_meter* modifies existing flow meter **action** based on **modify_field_select**. **modify_field_select** is according to the device specification. +Action: Sampler +*mlx5dv_dr_action_create_flow_sampler* creates a sampler action, allowing us to duplicate and sample a portion of traffic. +Packets steered to the sampler action will be sampled with an approximate probability of 1/sample_ratio provided in **attr**, and sample_actions provided in **attr** will be executed over them. +All original packets will be steered to default_next_table in **attr**. +A modify header format SET_ACTION data can be provided in action of **attr**, which can be executed on packets before going to default flow table. On some devices, this is required to set register value. + Action Flags: action **flags** can be set to one of the types of *enum mlx5dv_dr_action_flags*: **MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL**: is used to indicate the action is targeted for flow table in level=0 (ROOT) of the specific domain. diff --git a/providers/mlx5/mlx5.h b/providers/mlx5/mlx5.h index ee1ae51dc..ee66fde81 100644 --- a/providers/mlx5/mlx5.h +++ b/providers/mlx5/mlx5.h @@ -670,6 +670,7 @@ enum mlx5_devx_obj_type { MLX5_DEVX_TIR = 6, MLX5_DEVX_FLOW_GROUP = 7, MLX5_DEVX_FLOW_TABLE_ENTRY = 8, + MLX5_DEVX_FLOW_SAMPLER = 9, }; struct mlx5dv_devx_obj { diff --git a/providers/mlx5/mlx5_ifc.h b/providers/mlx5/mlx5_ifc.h index a986134dc..12fc41198 100644 --- a/providers/mlx5/mlx5_ifc.h +++ b/providers/mlx5/mlx5_ifc.h @@ -1865,6 +1865,7 @@ struct mlx5_ifc_alloc_flow_counter_out_bits { enum { MLX5_OBJ_TYPE_FLOW_METER = 0x000a, + MLX5_OBJ_TYPE_FLOW_SAMPLER = 0x0020, }; struct mlx5_ifc_general_obj_in_cmd_hdr_bits { @@ -1922,6 +1923,36 @@ struct mlx5_ifc_query_flow_meter_out_bits { struct mlx5_ifc_flow_meter_bits obj; }; +struct mlx5_ifc_flow_sampler_bits { + u8 modify_field_select[0x40]; + + u8 table_type[0x8]; + u8 level[0x8]; + u8 reserved_at_50[0xf]; + u8 ignore_flow_level[0x1]; + + u8 sample_ratio[0x20]; + + u8 reserved_at_80[0x8]; + u8 sample_table_id[0x18]; + + u8 reserved_at_a0[0x8]; + u8 default_table_id[0x18]; + + u8 sw_steering_icm_address_rx[0x40]; + u8 sw_steering_icm_address_tx[0x40]; +}; + +struct mlx5_ifc_create_flow_sampler_in_bits { + struct mlx5_ifc_general_obj_in_cmd_hdr_bits hdr; + struct mlx5_ifc_flow_sampler_bits sampler; +}; + +struct mlx5_ifc_query_flow_sampler_out_bits { + struct mlx5_ifc_general_obj_out_cmd_hdr_bits hdr; + struct mlx5_ifc_flow_sampler_bits obj; +}; + struct mlx5_ifc_esw_vport_context_bits { u8 reserved_at_0[0x3]; u8 vport_svlan_strip[0x1]; diff --git a/providers/mlx5/mlx5dv.h b/providers/mlx5/mlx5dv.h index a381431d5..232e22edf 100644 --- a/providers/mlx5/mlx5dv.h +++ b/providers/mlx5/mlx5dv.h @@ -1452,6 +1452,14 @@ struct mlx5dv_dr_flow_meter_attr { void *flow_meter_parameter; }; +struct mlx5dv_dr_flow_sampler_attr { + uint32_t sample_ratio; + struct mlx5dv_dr_table *default_next_table; + uint32_t num_sample_actions; + struct mlx5dv_dr_action **sample_actions; + __be64 action; +}; + struct mlx5dv_dr_domain * mlx5dv_dr_domain_create(struct ibv_context *ctx, enum mlx5dv_dr_domain_type type); @@ -1530,6 +1538,9 @@ int mlx5dv_dr_action_modify_flow_meter(struct mlx5dv_dr_action *action, struct mlx5dv_dr_flow_meter_attr *attr, __be64 modify_field_select); +struct mlx5dv_dr_action * +mlx5dv_dr_action_create_flow_sampler(struct mlx5dv_dr_flow_sampler_attr *attr); + int mlx5dv_dr_action_destroy(struct mlx5dv_dr_action *action); int mlx5dv_dump_dr_domain(FILE *fout, struct mlx5dv_dr_domain *domain); diff --git a/providers/mlx5/mlx5dv_dr.h b/providers/mlx5/mlx5dv_dr.h index 3064e9713..4bfbcbaff 100644 --- a/providers/mlx5/mlx5dv_dr.h +++ b/providers/mlx5/mlx5dv_dr.h @@ -145,6 +145,7 @@ enum dr_action_type { DR_ACTION_TYP_VPORT, DR_ACTION_TYP_METER, DR_ACTION_TYP_MISS, + DR_ACTION_TYP_SAMPLER, DR_ACTION_TYP_MAX, }; @@ -675,6 +676,15 @@ struct dr_devx_tbl { struct mlx5dv_devx_obj *fte_dvo; }; +struct dr_devx_flow_sampler_attr { + uint8_t table_type; + uint8_t level; + uint8_t ignore_flow_level; + uint32_t sample_ratio; + uint32_t default_next_table_id; + uint32_t sample_table_id; +}; + struct dr_domain_rx_tx { uint64_t drop_icm_addr; uint64_t default_icm_addr; @@ -768,6 +778,27 @@ struct dr_ste_action_modify_field { uint8_t l4_type; }; +struct dr_devx_tbl_with_refs { + uint16_t ref_actions_num; + struct mlx5dv_dr_action **ref_actions; + struct dr_devx_tbl *devx_tbl; +}; + +struct dr_flow_sampler { + struct mlx5dv_devx_obj *devx_obj; + uint64_t rx_icm_addr; + uint64_t tx_icm_addr; + struct mlx5dv_dr_table *next_ft; +}; + +struct dr_flow_sampler_restore_tbl { + struct mlx5dv_dr_table *tbl; + struct mlx5dv_dr_matcher *matcher; + struct mlx5dv_dr_rule *rule; + struct mlx5dv_dr_action **actions; + uint16_t num_of_actions; +}; + struct mlx5dv_dr_action { enum dr_action_type action_type; atomic_int refcount; @@ -805,6 +836,13 @@ struct mlx5dv_dr_action { uint64_t rx_icm_addr; uint64_t tx_icm_addr; } meter; + struct { + struct mlx5dv_dr_domain *dmn; + struct dr_devx_tbl_with_refs *term_tbl; + struct dr_flow_sampler *sampler_default; + struct dr_flow_sampler_restore_tbl *restore_tbl; + struct dr_flow_sampler *sampler_restore; + } sampler; struct mlx5dv_dr_table *dest_tbl; struct { struct mlx5dv_devx_obj *devx_obj; @@ -947,6 +985,11 @@ dr_devx_create_always_hit_ft(struct ibv_context *ctx, struct dr_devx_flow_group_attr *fg_attr, struct dr_devx_flow_fte_attr *fte_attr); void dr_devx_destroy_always_hit_ft(struct dr_devx_tbl *devx_tbl); +struct mlx5dv_devx_obj * +dr_devx_create_flow_sampler(struct ibv_context *ctx, + struct dr_devx_flow_sampler_attr *sampler_attr); +int dr_devx_query_flow_sampler(struct mlx5dv_devx_obj *obj, + uint64_t *rx_icm_addr, uint64_t *tx_icm_addr); struct mlx5dv_devx_obj *dr_devx_create_reformat_ctx(struct ibv_context *ctx, enum reformat_type rt, size_t reformat_size, diff --git a/providers/mlx5/verbs.c b/providers/mlx5/verbs.c index fecc356d7..465025065 100644 --- a/providers/mlx5/verbs.c +++ b/providers/mlx5/verbs.c @@ -4770,6 +4770,8 @@ static void set_devx_obj_info(const void *in, const void *out, obj_type = DEVX_GET(general_obj_in_cmd_hdr, in, obj_type); if (obj_type == MLX5_OBJ_TYPE_FLOW_METER) obj->type = MLX5_DEVX_FLOW_METER; + else if (obj_type == MLX5_OBJ_TYPE_FLOW_SAMPLER) + obj->type = MLX5_DEVX_FLOW_SAMPLER; obj->object_id = DEVX_GET(general_obj_out_cmd_hdr, out, obj_id); break;