Merge pull request #396 from ton31337/feature/add_socket_reuseport

Add SO_REUSEPORT support to socket
twitter · Oct 6, 2022 · 60aaf85 · 60aaf85
2 parents c3f433b + 3c9cec1
commit 60aaf85
Show file tree

Hide file tree

Showing 7 changed files with 40 additions and 2 deletions.
diff --git a/src/nc_conf.c b/src/nc_conf.c
@@ -78,6 +78,10 @@ static const struct command conf_commands[] = {
       conf_set_bool,
       offsetof(struct conf_pool, tcpkeepalive) },
 
+    { string("reuseport"),
+      conf_set_bool,
+      offsetof(struct conf_pool, reuseport) },
+
     { string("redis_auth"),
       conf_set_string,
       offsetof(struct conf_pool, redis_auth) },
@@ -202,6 +206,7 @@ conf_pool_init(struct conf_pool *cp, const struct string *name)
 
     cp->redis = CONF_UNSET_NUM;
     cp->tcpkeepalive = CONF_UNSET_NUM;
+    cp->reuseport = CONF_UNSET_NUM;
     cp->redis_db = CONF_UNSET_NUM;
     cp->preconnect = CONF_UNSET_NUM;
     cp->auto_eject_hosts = CONF_UNSET_NUM;
@@ -290,6 +295,7 @@ conf_pool_each_transform(void *elem, void *data)
     sp->hash_tag = cp->hash_tag;
 
     sp->tcpkeepalive = cp->tcpkeepalive ? 1 : 0;
+    sp->reuseport = cp->reuseport ? 1 : 0;
 
     sp->redis = cp->redis ? 1 : 0;
     sp->timeout = cp->timeout;
@@ -1245,6 +1251,10 @@ conf_validate_pool(struct conf *cf, struct conf_pool *cp)
         cp->tcpkeepalive = CONF_DEFAULT_TCPKEEPALIVE;
     }
 
+    if (cp->reuseport == CONF_UNSET_NUM) {
+	cp->reuseport = CONF_DEFAULT_REUSEPORT;
+    }
+
     if (cp->redis_db == CONF_UNSET_NUM) {
         cp->redis_db = CONF_DEFAULT_REDIS_DB;
     }

diff --git a/src/nc_conf.h b/src/nc_conf.h
@@ -55,6 +55,7 @@
 #define CONF_DEFAULT_SERVER_CONNECTIONS      1
 #define CONF_DEFAULT_KETAMA_PORT             11211
 #define CONF_DEFAULT_TCPKEEPALIVE            false
+#define CONF_DEFAULT_REUSEPORT		      false
 
 struct conf_listen {
     struct string   pname;   /* listen: as "hostname:port" */
@@ -95,6 +96,7 @@ struct conf_pool {
     int                server_failure_limit;  /* server_failure_limit: */
     struct array       server;                /* servers: conf_server[] */
     unsigned           valid:1;               /* valid? */
+    int                reuseport;             /* set SO_REUSEPORT to socket */
 };
 
 struct conf {

diff --git a/src/nc_proxy.c b/src/nc_proxy.c
@@ -142,6 +142,16 @@ proxy_listen(struct context *ctx, struct conn *p)
         return NC_ERROR;
     }
 
+    if (pool->reuseport) {
+        status = nc_set_reuseport(p->sd);
+        if (status < 0) {
+            log_error("reuse of port '%.*s' for listening on p %d failed: %s",
+                      pool->addrstr.len, pool->addrstr.data, p->sd,
+                      strerror(errno));
+            return NC_ERROR;
+        }
+    }
+
     status = bind(p->sd, p->addr, p->addrlen);
     if (status < 0) {
         log_error("bind on p %d to addr '%.*s' failed: %s", p->sd,
@@ -294,7 +304,7 @@ proxy_accept(struct context *ctx, struct conn *p)
                 return NC_OK;
             }
 
-            /* 
+            /*
              * Workaround of https://github.com/twitter/twemproxy/issues/97
              *
              * We should never reach here because the check for conn_ncurr_cconn()

diff --git a/src/nc_server.h b/src/nc_server.h
@@ -121,6 +121,7 @@ struct server_pool {
     unsigned           preconnect:1;         /* preconnect? */
     unsigned           redis:1;              /* redis? */
     unsigned           tcpkeepalive:1;       /* tcpkeepalive? */
+    unsigned           reuseport:1;          /* set SO_REUSEPORT to socket */
 };
 
 void server_ref(struct conn *conn, void *owner);

diff --git a/src/nc_stats.c b/src/nc_stats.c
@@ -828,7 +828,7 @@ stats_listen(struct stats *st)
         log_error("socket failed: %s", strerror(errno));
         return NC_ERROR;
     }
-
+    nc_set_reuseport(st->sd);
     status = nc_set_reuseaddr(st->sd);
     if (status < 0) {
         log_error("set reuseaddr on m %d failed for stats server: %s", st->sd, strerror(errno));

diff --git a/src/nc_util.c b/src/nc_util.c
@@ -75,6 +75,20 @@ nc_set_reuseaddr(int sd)
     return setsockopt(sd, SOL_SOCKET, SO_REUSEADDR, &reuse, len);
 }
 
+int
+nc_set_reuseport(int sd)
+{
+    int reuse;
+    socklen_t len;
+
+    reuse = 1;
+    len = sizeof(reuse);
+#ifdef SO_REUSEPORT
+    return setsockopt(sd, SOL_SOCKET, SO_REUSEPORT, &reuse, len);
+#endif
+    return -1;
+}
+
 /*
  * Disable Nagle algorithm on TCP socket.
  *

diff --git a/src/nc_util.h b/src/nc_util.h
@@ -93,6 +93,7 @@
 int nc_set_blocking(int sd);
 int nc_set_nonblocking(int sd);
 int nc_set_reuseaddr(int sd);
+int nc_set_reuseport(int sd);
 int nc_set_tcpnodelay(int sd);
 int nc_set_linger(int sd, int timeout);
 int nc_set_sndbuf(int sd, int size);