Skip to content

Commit

Permalink
mvpp2: add basic XDP support
Browse files Browse the repository at this point in the history 
Add XDP native support.
By now only XDP_DROP, XDP_PASS and XDP_REDIRECT
verdicts are supported.

Co-developed-by: Sven Auhagen <sven.auhagen@voleatech.de>
Signed-off-by: Sven Auhagen <sven.auhagen@voleatech.de>
Signed-off-by: Matteo Croce <mcroce@microsoft.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
  • Loading branch information
@teknoraver @davem330
teknoraver authored and davem330 committed Jul 3, 2020
1 parent b27db22 commit 07dd0a7
Show file tree
Hide file tree
Showing 2 changed files with 185 additions and 8 deletions.
28 changes: 23 additions & 5 deletions drivers/net/ethernet/marvell/mvpp2/mvpp2.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,18 @@
#include <linux/phylink.h>
#include <net/flow_offload.h>
#include <net/page_pool.h>
#include <linux/bpf.h>
#include <net/xdp.h>

/* The PacketOffset field is measured in units of 32 bytes and is 3 bits wide,
* so the maximum offset is 7 * 32 = 224
*/
#define MVPP2_SKB_HEADROOM min(max(XDP_PACKET_HEADROOM, NET_SKB_PAD), 224)

#define MVPP2_XDP_PASS 0
#define MVPP2_XDP_DROPPED BIT(0)
#define MVPP2_XDP_TX BIT(1)
#define MVPP2_XDP_REDIR BIT(2)

/* Fifo Registers */
#define MVPP2_RX_DATA_FIFO_SIZE_REG(port) (0x00 + 4 * (port))
Expand Down Expand Up @@ -629,10 +641,12 @@
ALIGN((mtu) + MVPP2_MH_SIZE + MVPP2_VLAN_TAG_LEN + \
ETH_HLEN + ETH_FCS_LEN, cache_line_size())

#define MVPP2_RX_BUF_SIZE(pkt_size) ((pkt_size) + NET_SKB_PAD)
#define MVPP2_RX_BUF_SIZE(pkt_size) ((pkt_size) + MVPP2_SKB_HEADROOM)
#define MVPP2_RX_TOTAL_SIZE(buf_size) ((buf_size) + MVPP2_SKB_SHINFO_SIZE)
#define MVPP2_RX_MAX_PKT_SIZE(total_size) \
((total_size) - NET_SKB_PAD - MVPP2_SKB_SHINFO_SIZE)
((total_size) - MVPP2_SKB_HEADROOM - MVPP2_SKB_SHINFO_SIZE)

#define MVPP2_MAX_RX_BUF_SIZE (PAGE_SIZE - MVPP2_SKB_SHINFO_SIZE - MVPP2_SKB_HEADROOM)

#define MVPP2_BIT_TO_BYTE(bit) ((bit) / 8)
#define MVPP2_BIT_TO_WORD(bit) ((bit) / 32)
Expand Down Expand Up @@ -690,9 +704,9 @@ enum mvpp2_prs_l3_cast {
#define MVPP2_BM_COOKIE_POOL_OFFS 8
#define MVPP2_BM_COOKIE_CPU_OFFS 24

#define MVPP2_BM_SHORT_FRAME_SIZE 512
#define MVPP2_BM_LONG_FRAME_SIZE 2048
#define MVPP2_BM_JUMBO_FRAME_SIZE 10240
#define MVPP2_BM_SHORT_FRAME_SIZE 704 /* frame size 128 */
#define MVPP2_BM_LONG_FRAME_SIZE 2240 /* frame size 1664 */
#define MVPP2_BM_JUMBO_FRAME_SIZE 10432 /* frame size 9856 */
/* BM short pool packet size
* These value assure that for SWF the total number
* of bytes allocated for each buffer will be 512
Expand Down Expand Up @@ -913,6 +927,8 @@ struct mvpp2_port {
unsigned int ntxqs;
struct net_device *dev;

struct bpf_prog *xdp_prog;

int pkt_size;

/* Per-CPU port control */
Expand All @@ -932,6 +948,8 @@ struct mvpp2_port {
struct mvpp2_pcpu_stats __percpu *stats;
u64 *ethtool_stats;

unsigned long state;

/* Per-port work and its lock to gather hardware statistics */
struct mutex gather_stats_lock;
struct delayed_work stats_work;
Expand Down
165 changes: 162 additions & 3 deletions drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
#include <net/ip.h>
#include <net/ipv6.h>
#include <net/tso.h>
#include <linux/bpf_trace.h>

#include "mvpp2.h"
#include "mvpp2_prs.h"
Expand Down Expand Up @@ -105,6 +106,7 @@ mvpp2_create_page_pool(struct device *dev, int num, int len)
.nid = NUMA_NO_NODE,
.dev = dev,
.dma_dir = DMA_FROM_DEVICE,
.offset = MVPP2_SKB_HEADROOM,
.max_len = len,
};

Expand Down Expand Up @@ -2462,7 +2464,7 @@ static int mvpp2_rxq_init(struct mvpp2_port *port,
put_cpu();

/* Set Offset */
mvpp2_rxq_offset_set(port, rxq->id, NET_SKB_PAD);
mvpp2_rxq_offset_set(port, rxq->id, MVPP2_SKB_HEADROOM);

/* Set coalescing pkts and time */
mvpp2_rx_pkts_coal_set(port, rxq);
Expand Down Expand Up @@ -3037,16 +3039,69 @@ static u32 mvpp2_skb_tx_csum(struct mvpp2_port *port, struct sk_buff *skb)
return MVPP2_TXD_L4_CSUM_NOT | MVPP2_TXD_IP_CSUM_DISABLE;
}

static int
mvpp2_run_xdp(struct mvpp2_port *port, struct mvpp2_rx_queue *rxq,
struct bpf_prog *prog, struct xdp_buff *xdp,
struct page_pool *pp)
{
unsigned int len, sync, err;
struct page *page;
u32 ret, act;

len = xdp->data_end - xdp->data_hard_start - MVPP2_SKB_HEADROOM;
act = bpf_prog_run_xdp(prog, xdp);

/* Due xdp_adjust_tail: DMA sync for_device cover max len CPU touch */
sync = xdp->data_end - xdp->data_hard_start - MVPP2_SKB_HEADROOM;
sync = max(sync, len);

switch (act) {
case XDP_PASS:
ret = MVPP2_XDP_PASS;
break;
case XDP_REDIRECT:
err = xdp_do_redirect(port->dev, xdp, prog);
if (unlikely(err)) {
ret = MVPP2_XDP_DROPPED;
page = virt_to_head_page(xdp->data);
page_pool_put_page(pp, page, sync, true);
} else {
ret = MVPP2_XDP_REDIR;
}
break;
default:
bpf_warn_invalid_xdp_action(act);
fallthrough;
case XDP_ABORTED:
trace_xdp_exception(port->dev, prog, act);
fallthrough;
case XDP_DROP:
page = virt_to_head_page(xdp->data);
page_pool_put_page(pp, page, sync, true);
ret = MVPP2_XDP_DROPPED;
break;
}

return ret;
}

/* Main rx processing */
static int mvpp2_rx(struct mvpp2_port *port, struct napi_struct *napi,
int rx_todo, struct mvpp2_rx_queue *rxq)
{
struct net_device *dev = port->dev;
struct bpf_prog *xdp_prog;
struct xdp_buff xdp;
int rx_received;
int rx_done = 0;
u32 xdp_ret = 0;
u32 rcvd_pkts = 0;
u32 rcvd_bytes = 0;

rcu_read_lock();

xdp_prog = READ_ONCE(port->xdp_prog);

/* Get number of received packets and clamp the to-do */
rx_received = mvpp2_rxq_received(port, rxq->id);
if (rx_todo > rx_received)
Expand All @@ -3061,7 +3116,7 @@ static int mvpp2_rx(struct mvpp2_port *port, struct napi_struct *napi,
dma_addr_t dma_addr;
phys_addr_t phys_addr;
u32 rx_status;
int pool, rx_bytes, err;
int pool, rx_bytes, err, ret;
void *data;

rx_done++;
Expand Down Expand Up @@ -3097,6 +3152,33 @@ static int mvpp2_rx(struct mvpp2_port *port, struct napi_struct *napi,
else
frag_size = bm_pool->frag_size;

if (xdp_prog) {
xdp.data_hard_start = data;
xdp.data = data + MVPP2_MH_SIZE + MVPP2_SKB_HEADROOM;
xdp.data_end = xdp.data + rx_bytes;
xdp.frame_sz = PAGE_SIZE;

if (bm_pool->pkt_size == MVPP2_BM_SHORT_PKT_SIZE)
xdp.rxq = &rxq->xdp_rxq_short;
else
xdp.rxq = &rxq->xdp_rxq_long;

xdp_set_data_meta_invalid(&xdp);

ret = mvpp2_run_xdp(port, rxq, xdp_prog, &xdp, pp);

if (ret) {
xdp_ret |= ret;
err = mvpp2_rx_refill(port, bm_pool, pp, pool);
if (err) {
netdev_err(port->dev, "failed to refill BM pools\n");
goto err_drop_frame;
}

continue;
}
}

skb = build_skb(data, frag_size);
if (!skb) {
netdev_warn(port->dev, "skb build failed\n");
Expand All @@ -3119,7 +3201,7 @@ static int mvpp2_rx(struct mvpp2_port *port, struct napi_struct *napi,
rcvd_pkts++;
rcvd_bytes += rx_bytes;

skb_reserve(skb, MVPP2_MH_SIZE + NET_SKB_PAD);
skb_reserve(skb, MVPP2_MH_SIZE + MVPP2_SKB_HEADROOM);
skb_put(skb, rx_bytes);
skb->protocol = eth_type_trans(skb, dev);
mvpp2_rx_csum(port, rx_status, skb);
Expand All @@ -3134,6 +3216,8 @@ static int mvpp2_rx(struct mvpp2_port *port, struct napi_struct *napi,
mvpp2_bm_pool_put(port, pool, dma_addr, phys_addr);
}

rcu_read_unlock();

if (rcvd_pkts) {
struct mvpp2_pcpu_stats *stats = this_cpu_ptr(port->stats);

Expand Down Expand Up @@ -3609,13 +3693,17 @@ static void mvpp2_start_dev(struct mvpp2_port *port)
}

netif_tx_start_all_queues(port->dev);

clear_bit(0, &port->state);
}

/* Set hw internals when stopping port */
static void mvpp2_stop_dev(struct mvpp2_port *port)
{
int i;

set_bit(0, &port->state);

/* Disable interrupts on all threads */
mvpp2_interrupts_disable(port);

Expand Down Expand Up @@ -4022,6 +4110,10 @@ static int mvpp2_change_mtu(struct net_device *dev, int mtu)
}

if (MVPP2_RX_PKT_SIZE(mtu) > MVPP2_BM_LONG_PKT_SIZE) {
if (port->xdp_prog) {
netdev_err(dev, "Jumbo frames are not supported with XDP\n");
return -EINVAL;
}
if (priv->percpu_pools) {
netdev_warn(dev, "mtu %d too high, switching to shared buffers", mtu);
mvpp2_bm_switch_buffers(priv, false);
Expand Down Expand Up @@ -4160,6 +4252,72 @@ static int mvpp2_set_features(struct net_device *dev,
return 0;
}

static int mvpp2_xdp_setup(struct mvpp2_port *port, struct netdev_bpf *bpf)
{
struct bpf_prog *prog = bpf->prog, *old_prog;
bool running = netif_running(port->dev);
bool reset = !prog != !port->xdp_prog;

if (port->dev->mtu > ETH_DATA_LEN) {
NL_SET_ERR_MSG_MOD(bpf->extack, "XDP is not supported with jumbo frames enabled");
return -EOPNOTSUPP;
}

if (!port->priv->percpu_pools) {
NL_SET_ERR_MSG_MOD(bpf->extack, "Per CPU Pools required for XDP");
return -EOPNOTSUPP;
}

/* device is up and bpf is added/removed, must setup the RX queues */
if (running && reset) {
mvpp2_stop_dev(port);
mvpp2_cleanup_rxqs(port);
mvpp2_cleanup_txqs(port);
}

old_prog = xchg(&port->xdp_prog, prog);
if (old_prog)
bpf_prog_put(old_prog);

/* bpf is just replaced, RXQ and MTU are already setup */
if (!reset)
return 0;

/* device was up, restore the link */
if (running) {
int ret = mvpp2_setup_rxqs(port);

if (ret) {
netdev_err(port->dev, "mvpp2_setup_rxqs failed\n");
return ret;
}
ret = mvpp2_setup_txqs(port);
if (ret) {
netdev_err(port->dev, "mvpp2_setup_txqs failed\n");
return ret;
}

mvpp2_start_dev(port);
}

return 0;
}

static int mvpp2_xdp(struct net_device *dev, struct netdev_bpf *xdp)
{
struct mvpp2_port *port = netdev_priv(dev);

switch (xdp->command) {
case XDP_SETUP_PROG:
return mvpp2_xdp_setup(port, xdp);
case XDP_QUERY_PROG:
xdp->prog_id = port->xdp_prog ? port->xdp_prog->aux->id : 0;
return 0;
default:
return -EINVAL;
}
}

/* Ethtool methods */

static int mvpp2_ethtool_nway_reset(struct net_device *dev)
Expand Down Expand Up @@ -4510,6 +4668,7 @@ static const struct net_device_ops mvpp2_netdev_ops = {
.ndo_vlan_rx_add_vid = mvpp2_vlan_rx_add_vid,
.ndo_vlan_rx_kill_vid = mvpp2_vlan_rx_kill_vid,
.ndo_set_features = mvpp2_set_features,
.ndo_bpf = mvpp2_xdp,
};

static const struct ethtool_ops mvpp2_eth_tool_ops = {
Expand Down

0 comments on commit 07dd0a7

Please sign in to comment.