common: add common API 'get_numa_nodes' for numa-aware resource alloc…

…ation

include/common.h

@@ -107,6 +107,8 @@ enum {
 
 extern const char *dpvs_strerror(int err);
 
+int get_numa_nodes(void);
+
 int linux_set_if_mac(const char *ifname, const unsigned char mac[ETH_ALEN]);
 int linux_hw_mc_add(const char *ifname, const uint8_t hwma[ETH_ALEN]);
 int linux_hw_mc_del(const char *ifname, const uint8_t hwma[ETH_ALEN]);

src/common.c

@@ -21,6 +21,7 @@
 #include <errno.h>
 #include <unistd.h>
 #include <stdbool.h>
+#include <numa.h>
 #include <sys/ioctl.h>
 #include <sys/socket.h>
 #include <net/if.h>
@@ -85,6 +86,18 @@ dpvs_state_t dpvs_state_get(void)
     return g_dpvs_tate;
 }
 
+int get_numa_nodes(void)
+{
+    int numa_nodes;
+
+    if (numa_available() < 0)
+        numa_nodes = 0;
+    else
+        numa_nodes = numa_max_node();
+
+    return (numa_nodes + 1);
+}
+
 /* if (num+offset) == 2^n, return true,
  * otherwise return false and 'lower' is filled with
  * the closest lower bound value to 'num' */

src/config.mk

@@ -21,7 +21,7 @@
 # TODO: use standard way to define compile flags.
 #
 
-CFLAGS += -D DPVS_MAX_SOCKET=1
+CFLAGS += -D DPVS_MAX_SOCKET=2
 CFLAGS += -D DPVS_MAX_LCORE=64
 
 #CFLAGS += -D CONFIG_RECORD_BIG_LOOP

src/ipvs/ip_vs_conn.c

@@ -1480,7 +1480,7 @@ int dp_vs_conn_init(void)
     conn_ctrl_init();
 
     /* connection cache on each NUMA socket */
-    for (i = 0; i < DPVS_MAX_SOCKET; i++) {
+    for (i = 0; i < get_numa_nodes(); i++) {
         snprintf(poolname, sizeof(poolname), "dp_vs_conn_%d", i);
         dp_vs_conn_cache[i] = rte_mempool_create(poolname,
                                     conn_pool_size,

src/ipvs/ip_vs_synproxy.c

@@ -124,7 +124,7 @@ int dp_vs_synproxy_init(void)
     dpvs_timer_sched(&g_minute_timer, &tv, minute_timer_expire, NULL, true);
 
     /* allocate NUMA-aware ACK list cache */
-    for (i = 0; i < DPVS_MAX_SOCKET; i++) {
+    for (i = 0; i < get_numa_nodes(); i++) {
         snprintf(ack_mbufpool_name, sizeof(ack_mbufpool_name), "ack_mbufpool_%d", i);
         dp_vs_synproxy_ack_mbufpool[i] = rte_mempool_create(ack_mbufpool_name,
                 DP_VS_SYNPROXY_ACK_MBUFPOOL_SIZE,
@@ -153,7 +153,7 @@ int dp_vs_synproxy_term(void)
     int i;
     dpvs_timer_cancel(&g_minute_timer, true);
 
-    for (i = 0; i < DPVS_MAX_SOCKET; i++)
+    for (i = 0; i < get_numa_nodes(); i++)
         rte_mempool_free(dp_vs_synproxy_ack_mbufpool[i]);
 
     return EDPVS_OK;

src/kni.c

@@ -382,7 +382,7 @@ int kni_init(void)
     int i;
     char poolname[32];
 
-    for (i = 0; i < DPVS_MAX_SOCKET; i++) {
+    for (i = 0; i < get_numa_nodes(); i++) {
         memset(poolname, 0, sizeof(poolname));
         snprintf(poolname, sizeof(poolname) - 1, "kni_mbuf_pool_%d", i);
 

src/main.c

@@ -16,7 +16,6 @@
  *
  */
 #define _GNU_SOURCE
-#include <numa.h>
 #include <pthread.h>
 #include <assert.h>
 #include "pidfile.h"
@@ -76,7 +75,7 @@ static int set_all_thread_affinity(void)
 
 int main(int argc, char *argv[])
 {
-    int err, nports, numa_nodes;
+    int err, nports;
     portid_t pid;
     struct netif_port *dev;
     struct timeval tv;
@@ -96,14 +95,8 @@ int main(int argc, char *argv[])
     gettimeofday(&tv, NULL);
     srandom(tv.tv_sec ^ tv.tv_usec ^ getpid());
 
-    if (numa_available() < 0)
-        numa_nodes = 0;
-    else
-        numa_nodes = numa_max_node();
-    numa_nodes++;
-
-    if (numa_nodes != DPVS_MAX_SOCKET) {
-        fprintf(stderr, "DPVS_MAX_SOCKET and system numa nodes do not match!\n");
+    if (get_numa_nodes() > DPVS_MAX_SOCKET) {
+        fprintf(stderr, "DPVS_MAX_SOCKET is smaller than system numa nodes!\n");
         return -1;
     }
 

src/netif.c

@@ -879,7 +879,7 @@ static inline void netif_pktmbuf_pool_init(void)
 {
     int i;
     char poolname[32];
-    for (i = 0; i < DPVS_MAX_SOCKET; i++) {
+    for (i = 0; i < get_numa_nodes(); i++) {
         snprintf(poolname, sizeof(poolname), "mbuf_pool_%d", i);
         pktmbuf_pool[i] = rte_pktmbuf_pool_create(poolname, netif_pktpool_nb_mbuf,
                 netif_pktpool_mbuf_cache, 0, RTE_MBUF_DEFAULT_BUF_SIZE, i);

src/tc/tc.c

@@ -362,7 +362,7 @@ int tc_init(void)
     tc_register_cls(&match_cls_ops);
 
     /* per-NUMA socket mempools for queued tc_mbuf{} */
-    for (s = 0; s < DPVS_MAX_SOCKET; s++) {
+    for (s = 0; s < get_numa_nodes(); s++) {
         char plname[64];
 
         snprintf(plname, sizeof(plname), "tc_mbuf_pool_%d", s);

tools/dpip/Makefile

@@ -27,8 +27,10 @@ CFLAGS += -Wall -Werror -Wstrict-prototypes -Wmissing-prototypes
 CFLAGS += -I ../../include
 CFLAGS += -I ../keepalived/keepalived/libipvs-2.6
 
+LIBS = -lnuma
 DEFS = -D DPVS_MAX_LCORE=64
-CFLAGS += $(DEFS)
+
+CFLAGS += $(LIBS) $(DEFS)
 
 OBJS = dpip.o utils.o route.o addr.o neigh.o link.o vlan.o \
 	   qsch.o cls.o tunnel.o ../../src/common.o \

tools/ipvsadm/Makefile

@@ -83,7 +83,7 @@ POPT_DEFINE = -DHAVE_POPT
 endif
 
 OBJS		= ipvsadm.o config_stream.o dynamic_array.o
-LIBS		= $(POPT_LIB)
+LIBS		= $(POPT_LIB) -lnuma
 DEFINES		= -DVERSION=\"$(VERSION)\" -DSCHEDULERS=\"$(SCHEDULERS)\" \
 		  -DPE_LIST=\"$(PE_LIST)\" $(POPT_DEFINE)
 DEFINES		+= $(shell if [ ! -f ../ip_vs.h ]; then	\

tools/keepalived/keepalived/Makefile.in

@@ -22,7 +22,7 @@ sysconf_dir = $(sysconfdir)/sysconfig
 
 CC = @CC@
 STRIP = @STRIP@
-LDFLAGS = @LIBS@ @LDFLAGS@ -ldl
+LDFLAGS = @LIBS@ @LDFLAGS@ -ldl -lnuma
 SUBDIRS = core
 DPVSDEPS = ../../../src/common.o