Skip to content


Browse files Browse the repository at this point in the history
epoll: Add busy poll support to epoll with socket fds.
This patch adds busy poll support to epoll. The implementation is meant to
be opportunistic in that it will take the NAPI ID from the last socket
that is added to the ready list that contains a valid NAPI ID and it will
use that for busy polling until the ready list goes empty.  Once the ready
list goes empty the NAPI ID is reset and busy polling is disabled until a
new socket is added to the ready list.

In addition when we insert a new socket into the epoll we record the NAPI
ID and assume we are going to receive events on it.  If that doesn't occur
it will be evicted as the active NAPI ID and we will resume normal

An application can use SO_INCOMING_CPU or SO_REUSEPORT_ATTACH_C/EBPF socket
options to spread the incoming connections to specific worker threads
based on the incoming queue. This enables epoll for each worker thread
to have only sockets that receive packets from a single queue. So when an
application calls epoll_wait() and there are no events available to report,
busy polling is done on the associated queue to pull the packets.

Signed-off-by: Sridhar Samudrala <>
Signed-off-by: Alexander Duyck <>
Acked-by: Eric Dumazet <>
Signed-off-by: David S. Miller <>
  • Loading branch information
ssamudrala authored and davem330 committed Mar 25, 2017
1 parent 7db6b04 commit bf3b9f6
Showing 1 changed file with 93 additions and 0 deletions.
93 changes: 93 additions & 0 deletions fs/eventpoll.c
Expand Up @@ -42,6 +42,7 @@
#include <linux/seq_file.h>
#include <linux/compat.h>
#include <linux/rculist.h>
#include <net/busy_poll.h>

Expand Down Expand Up @@ -224,6 +225,11 @@ struct eventpoll {
/* used to optimize loop detection check */
int visited;
struct list_head visited_list_link;

/* used to track busy poll napi_id */
unsigned int napi_id;

/* Wait structure used by the poll hooks */
Expand Down Expand Up @@ -384,6 +390,77 @@ static inline int ep_events_available(struct eventpoll *ep)
return !list_empty(&ep->rdllist) || ep->ovflist != EP_UNACTIVE_PTR;

static bool ep_busy_loop_end(void *p, unsigned long start_time)
struct eventpoll *ep = p;

return ep_events_available(ep) || busy_loop_timeout(start_time);

* Busy poll if globally on and supporting sockets found && no events,
* busy loop will return if need_resched or ep_events_available.
* we must do our busy polling with irqs enabled
static void ep_busy_loop(struct eventpoll *ep, int nonblock)
unsigned int napi_id = READ_ONCE(ep->napi_id);

if ((napi_id >= MIN_NAPI_ID) && net_busy_loop_on())
napi_busy_loop(napi_id, nonblock ? NULL : ep_busy_loop_end, ep);

static inline void ep_reset_busy_poll_napi_id(struct eventpoll *ep)
if (ep->napi_id)
ep->napi_id = 0;

* Set epoll busy poll NAPI ID from sk.
static inline void ep_set_busy_poll_napi_id(struct epitem *epi)
struct eventpoll *ep;
unsigned int napi_id;
struct socket *sock;
struct sock *sk;
int err;

if (!net_busy_loop_on())

sock = sock_from_file(epi->ffd.file, &err);
if (!sock)

sk = sock->sk;
if (!sk)

napi_id = READ_ONCE(sk->sk_napi_id);
ep = epi->ep;

/* Non-NAPI IDs can be rejected
* or
* Nothing to do if we already have this ID
if (napi_id < MIN_NAPI_ID || napi_id == ep->napi_id)

/* record NAPI ID for use in next busy poll */
ep->napi_id = napi_id;

* ep_call_nested - Perform a bound (possibly) nested call, by checking
* that the recursion limit is not exceeded, and that
Expand Down Expand Up @@ -1022,6 +1099,8 @@ static int ep_poll_callback(wait_queue_t *wait, unsigned mode, int sync, void *k

spin_lock_irqsave(&ep->lock, flags);


* If the event mask does not contain any poll(2) event, we consider the
* descriptor to be disabled. This condition is likely the effect of the
Expand Down Expand Up @@ -1363,6 +1442,9 @@ static int ep_insert(struct eventpoll *ep, struct epoll_event *event,
/* We have to drop the new item inside our item list to keep track of it */
spin_lock_irqsave(&ep->lock, flags);

/* record NAPI ID of new item if present */

/* If the file is already "ready" we drop it inside the ready list */
if ((revents & event->events) && !ep_is_linked(&epi->rdllink)) {
list_add_tail(&epi->rdllink, &ep->rdllist);
Expand Down Expand Up @@ -1637,9 +1719,20 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events,


if (!ep_events_available(ep))
ep_busy_loop(ep, timed_out);

spin_lock_irqsave(&ep->lock, flags);

if (!ep_events_available(ep)) {
* Busy poll timed out. Drop NAPI ID for now, we can add
* it back in when we have moved a socket with a valid NAPI
* ID onto the ready list.

* We don't have any available event to return to the caller.
* We need to sleep here, and we will be wake up by
Expand Down

0 comments on commit bf3b9f6

Please sign in to comment.