net/mlx5e: Use select queue parameters to sync with control flow

Start using the select queue parameters introduced in the previous
commit to have proper synchronization with changing the configuration
(such as number of channels and queues). It ensures that the state that
mlx5e_select_queue() sees is always consistent and stays the same while
the function is running. Also it allows mlx5e_select_queue to stop using
data structures that weren't synchronized properly: txq2sq,
channel_tc2realtxq, port_ptp_tc2realtxq. The last two are removed
completely, as they were used only in mlx5e_select_queue.

Issue: 2392615
Change-Id: Ifc4f593ac6b438813dd01bbff7e91e0f0083e6d6
Signed-off-by: Maxim Mikityanskiy <maximmi@mellanox.com>
Reviewed-by: Tariq Toukan <tariqt@nvidia.com>
Signed-off-by: Saeed Mahameed <saeedm@nvidia.com>

drivers/net/ethernet/mellanox/mlx5/core/en.h

@@ -879,8 +879,6 @@ struct mlx5e_priv {
 	/* priv data path fields - start */
 	struct mlx5e_selq selq;
 	struct mlx5e_txqsq **txq2sq;
-	int **channel_tc2realtxq;
-	int port_ptp_tc2realtxq[MLX5E_MAX_NUM_TC];
 #ifdef CONFIG_MLX5_CORE_EN_DCB
 	struct mlx5e_dcbx_dp       dcbx_dp;
 #endif
@@ -923,7 +921,6 @@ struct mlx5e_priv {
 	u16                        drop_rq_q_counter;
 	struct notifier_block      events_nb;
 	struct notifier_block      blocking_events_nb;
-	int                        num_tc_x_num_ch;
 
 	struct udp_tunnel_nic_info nic_info;
 #ifdef CONFIG_MLX5_CORE_EN_DCB

drivers/net/ethernet/mellanox/mlx5/core/en/qos.c

@@ -549,11 +549,15 @@ int mlx5e_htb_root_del(struct mlx5e_priv *priv)
 
 	qos_dbg(priv->mdev, "TC_HTB_DESTROY\n");
 
+	/* Wait until real_num_tx_queues is updated for mlx5e_select_queue,
+	 * so that we can safely switch to its non-HTB non-PTP fastpath.
+	 */
+	synchronize_net();
+
 	mlx5e_selq_prepare(&priv->selq, &priv->channels.params, false);
 	mlx5e_selq_apply(&priv->selq);
 
 	WRITE_ONCE(priv->htb.maj_id, 0);
-	synchronize_rcu(); /* Sync with mlx5e_select_htb_queue and TX data path. */
 
 	root = mlx5e_sw_node_find(priv, MLX5E_HTB_CLASSID_ROOT);
 	if (!root) {

drivers/net/ethernet/mellanox/mlx5/core/en/selq.c

@@ -109,12 +109,13 @@ static int mlx5e_get_dscp_up(struct mlx5e_priv *priv, struct sk_buff *skb)
 }
 #endif
 
-static u16 mlx5e_select_ptpsq(struct net_device *dev, struct sk_buff *skb)
+static u16 mlx5e_select_ptpsq(struct net_device *dev, struct sk_buff *skb,
+			      struct mlx5e_selq_params *selq)
 {
 	struct mlx5e_priv *priv = netdev_priv(dev);
 	int up = 0;
 
-	if (!netdev_get_num_tc(dev))
+	if (selq->num_tcs <= 1)
 		goto return_txq;
 
 #ifdef CONFIG_MLX5_CORE_EN_DCB
@@ -126,15 +127,15 @@ static u16 mlx5e_select_ptpsq(struct net_device *dev, struct sk_buff *skb)
 			up = skb_vlan_tag_get_prio(skb);
 
 return_txq:
-	return priv->port_ptp_tc2realtxq[up];
+	return selq->num_regular_queues + up;
 }
 
-static int mlx5e_select_htb_queue(struct mlx5e_priv *priv, struct sk_buff *skb,
-				  u16 htb_maj_id)
+static int mlx5e_select_htb_queue(struct mlx5e_priv *priv, struct sk_buff *skb)
 {
 	u16 classid;
 
-	if ((TC_H_MAJ(skb->priority) >> 16) == htb_maj_id)
+	/* Order maj_id before defcls - pairs with mlx5e_htb_root_add. */
+	if ((TC_H_MAJ(skb->priority) >> 16) == smp_load_acquire(&priv->htb.maj_id))
 		classid = TC_H_MIN(skb->priority);
 	else
 		classid = READ_ONCE(priv->htb.defcls);
@@ -149,44 +150,42 @@ u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb,
 		       struct net_device *sb_dev)
 {
 	struct mlx5e_priv *priv = netdev_priv(dev);
-	int num_tc_x_num_ch;
+	struct mlx5e_selq_params *selq;
 	int txq_ix;
 	int up = 0;
-	int ch_ix;
 
-	/* Sync with mlx5e_update_num_tc_x_num_ch - avoid refetching. */
-	num_tc_x_num_ch = READ_ONCE(priv->num_tc_x_num_ch);
-	if (unlikely(dev->real_num_tx_queues > num_tc_x_num_ch)) {
-		struct mlx5e_ptp *ptp_channel;
+	selq = mlx5e_selq_get_params(&priv->selq);
 
-		/* Order maj_id before defcls - pairs with mlx5e_htb_root_add. */
-		u16 htb_maj_id = smp_load_acquire(&priv->htb.maj_id);
+	/* This is a workaround needed only for the mlx5e_netdev_change_profile
+	 * flow that zeroes out the whole priv without unregistering the netdev
+	 * and without preventing ndo_select_queue from being called.
+	 */
+	if (unlikely(!selq))
+		return 0;
 
-		if (unlikely(htb_maj_id)) {
-			txq_ix = mlx5e_select_htb_queue(priv, skb, htb_maj_id);
+	if (unlikely(selq->is_ptp || selq->is_htb)) {
+		if (unlikely(selq->is_htb)) {
+			txq_ix = mlx5e_select_htb_queue(priv, skb);
 			if (txq_ix > 0)
 				return txq_ix;
 		}
 
-		ptp_channel = READ_ONCE(priv->channels.ptp);
-		if (unlikely(ptp_channel &&
-			     test_bit(MLX5E_PTP_STATE_TX, ptp_channel->state) &&
-			     mlx5e_use_ptpsq(skb)))
-			return mlx5e_select_ptpsq(dev, skb);
+		if (unlikely(selq->is_ptp && mlx5e_use_ptpsq(skb)))
+			return mlx5e_select_ptpsq(dev, skb, selq);
 
 		txq_ix = netdev_pick_tx(dev, skb, NULL);
 		/* Fix netdev_pick_tx() not to choose ptp_channel and HTB txqs.
 		 * If they are selected, switch to regular queues.
 		 * Driver to select these queues only at mlx5e_select_ptpsq()
 		 * and mlx5e_select_htb_queue().
 		 */
-		if (unlikely(txq_ix >= num_tc_x_num_ch))
-			txq_ix %= num_tc_x_num_ch;
+		if (unlikely(txq_ix >= selq->num_regular_queues))
+			txq_ix %= selq->num_regular_queues;
 	} else {
 		txq_ix = netdev_pick_tx(dev, skb, NULL);
 	}
 
-	if (!netdev_get_num_tc(dev))
+	if (selq->num_tcs <= 1)
 		return txq_ix;
 
 #ifdef CONFIG_MLX5_CORE_EN_DCB
@@ -201,7 +200,5 @@ u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb,
 	 * So we can return a txq_ix that matches the channel and
 	 * packet UP.
 	 */
-	ch_ix = priv->txq2sq[txq_ix]->ch_ix;
-
-	return priv->channel_tc2realtxq[ch_ix][up];
+	return txq_ix % selq->num_channels + up * selq->num_channels;
 }

drivers/net/ethernet/mellanox/mlx5/core/en/selq.h

@@ -5,6 +5,7 @@
 #define __MLX5_EN_SELQ_H__
 
 #include <linux/kernel.h>
+#include <linux/rcupdate.h>
 
 struct mlx5e_selq_params;
 
@@ -15,6 +16,11 @@ struct mlx5e_selq {
 	bool is_prepared;
 };
 
+static inline struct mlx5e_selq_params *mlx5e_selq_get_params(struct mlx5e_selq *selq)
+{
+	return rcu_dereference_bh(selq->active);
+}
+
 struct mlx5e_params;
 struct net_device;
 struct sk_buff;

drivers/net/ethernet/mellanox/mlx5/core/en_main.c

@@ -2677,7 +2677,6 @@ static void mlx5e_build_txq_maps(struct mlx5e_priv *priv)
 			struct mlx5e_txqsq *sq = &c->sq[tc];
 
 			priv->txq2sq[sq->txq_ix] = sq;
-			priv->channel_tc2realtxq[i][tc] = i + tc * ch;
 		}
 	}
 
@@ -2692,7 +2691,6 @@ static void mlx5e_build_txq_maps(struct mlx5e_priv *priv)
 		struct mlx5e_txqsq *sq = &c->ptpsq[tc].txqsq;
 
 		priv->txq2sq[sq->txq_ix] = sq;
-		priv->port_ptp_tc2realtxq[tc] = priv->num_tc_x_num_ch + tc;
 	}
 
 out:
@@ -2703,16 +2701,8 @@ static void mlx5e_build_txq_maps(struct mlx5e_priv *priv)
 	smp_wmb();
 }
 
-static void mlx5e_update_num_tc_x_num_ch(struct mlx5e_priv *priv)
-{
-	/* Sync with mlx5e_select_queue. */
-	WRITE_ONCE(priv->num_tc_x_num_ch,
-		   mlx5e_get_dcb_num_tc(&priv->channels.params) * priv->channels.num);
-}
-
 void mlx5e_activate_priv_channels(struct mlx5e_priv *priv)
 {
-	mlx5e_update_num_tc_x_num_ch(priv);
 	mlx5e_build_txq_maps(priv);
 	mlx5e_activate_channels(&priv->channels);
 	mlx5e_qos_activate_queues(priv);
@@ -4667,11 +4657,6 @@ void mlx5e_build_nic_params(struct mlx5e_priv *priv, struct mlx5e_xsk *xsk, u16
 				     priv->max_nch);
 	mlx5e_params_mqprio_reset(params);
 
-	/* Set an initial non-zero value, so that mlx5e_select_queue won't
-	 * divide by zero if called before first activating channels.
-	 */
-	priv->num_tc_x_num_ch = params->num_channels * params->mqprio.num_tc;
-
 	/* SQ */
 	params->log_sq_size = is_kdump_kernel() ?
 		MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE :
@@ -5224,7 +5209,7 @@ int mlx5e_priv_init(struct mlx5e_priv *priv,
 		    struct net_device *netdev,
 		    struct mlx5_core_dev *mdev)
 {
-	int nch, num_txqs, node, i;
+	int nch, num_txqs, node;
 	int err;
 
 	num_txqs = netdev->num_tx_queues;
@@ -5265,30 +5250,13 @@ int mlx5e_priv_init(struct mlx5e_priv *priv,
 	if (!priv->tx_rates)
 		goto err_free_txq2sq;
 
-	priv->channel_tc2realtxq =
-		kcalloc_node(nch, sizeof(*priv->channel_tc2realtxq), GFP_KERNEL, node);
-	if (!priv->channel_tc2realtxq)
-		goto err_free_tx_rates;
-
-	for (i = 0; i < nch; i++) {
-		priv->channel_tc2realtxq[i] =
-			kcalloc_node(profile->max_tc, sizeof(**priv->channel_tc2realtxq),
-				     GFP_KERNEL, node);
-		if (!priv->channel_tc2realtxq[i])
-			goto err_free_channel_tc2realtxq;
-	}
-
 	priv->channel_stats =
 		kcalloc_node(nch, sizeof(*priv->channel_stats), GFP_KERNEL, node);
 	if (!priv->channel_stats)
-		goto err_free_channel_tc2realtxq;
+		goto err_free_tx_rates;
 
 	return 0;
 
-err_free_channel_tc2realtxq:
-	while (--i >= 0)
-		kfree(priv->channel_tc2realtxq[i]);
-	kfree(priv->channel_tc2realtxq);
 err_free_tx_rates:
 	kfree(priv->tx_rates);
 err_free_txq2sq:
@@ -5313,9 +5281,6 @@ void mlx5e_priv_cleanup(struct mlx5e_priv *priv)
 	for (i = 0; i < priv->stats_nch; i++)
 		kvfree(priv->channel_stats[i]);
 	kfree(priv->channel_stats);
-	for (i = 0; i < priv->max_nch; i++)
-		kfree(priv->channel_tc2realtxq[i]);
-	kfree(priv->channel_tc2realtxq);
 	kfree(priv->tx_rates);
 	kfree(priv->txq2sq);
 	destroy_workqueue(priv->wq);

drivers/net/ethernet/mellanox/mlx5/core/en_rep.c

@@ -632,11 +632,6 @@ static void mlx5e_build_rep_params(struct net_device *netdev)
 	params->mqprio.num_tc       = 1;
 	params->tunneled_offload_en = false;
 
-	/* Set an initial non-zero value, so that mlx5e_select_queue won't
-	 * divide by zero if called before first activating channels.
-	 */
-	priv->num_tc_x_num_ch = params->num_channels * params->mqprio.num_tc;
-
 	mlx5_query_min_inline(mdev, &params->tx_min_inline_mode);
 }